From 23568691803da584507d05b83cb95eb90cabcba0 Mon Sep 17 00:00:00 2001 From: Stepan Blyshchak Date: Tue, 3 Nov 2020 12:41:24 +0200 Subject: [PATCH 001/173] [sonic-installer] migrate SONiC packages Signed-off-by: Stepan Blyshchak --- sonic_installer/main.py | 95 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 85 insertions(+), 10 deletions(-) diff --git a/sonic_installer/main.py b/sonic_installer/main.py index ca843c394b..43da10f11f 100644 --- a/sonic_installer/main.py +++ b/sonic_installer/main.py @@ -213,17 +213,48 @@ def print_deprecation_warning(deprecated_cmd_or_subcmd, new_cmd_or_subcmd): fg="red", err=True) click.secho("Please use '{}' instead".format(new_cmd_or_subcmd), fg="red", err=True) + +def mount_squash_fs(squashfs_path, mount_point): + run_command_or_raise(["mkdir", "-p", mount_point]) + run_command_or_raise(["mount", "-t", "squashfs", squashfs_path, mount_point]) + + +def umount(mount_point, read_only=True, recursive=False, force=True, remove_dir=True): + flags = "-" + if read_only: + flags = flags + "r" + if force: + flags = flags + "f" + if recursive: + flags = flags + "R" + run_command_or_raise(["umount", flags, mount_point]) + if remove_dir: + run_command_or_raise(["rm", "-rf", mount_point]) + + +def mount_overlay_fs(lowerdir, upperdir, workdir, mount_point): + run_command_or_raise(["mkdir", "-p", mount_point]) + overlay_options = "rw,relatime,lowerdir={},upperdir={},workdir={}".format(lowerdir, upperdir, workdir) + run_command_or_raise(["mount", "overlay", "-t", "overlay", "-o", overlay_options, mount_point]) + + +def mount_bind(source, mount_point): + run_command_or_raise(["mkdir", "-p", mount_point]) + run_command_or_raise(["mount", "--bind", source, mount_point]) + + +def mount_procfs_chroot(root): + run_command_or_raise(["chroot", root, "mount", "proc", "/proc", "-t", "proc"]) + + +def mount_sysfs_chroot(root): + run_command_or_raise(["chroot", root, "mount", "sysfs", "/sys", "-t", "sysfs"]) + + def update_sonic_environment(click, binary_image_version): """Prepare sonic environment variable using incoming image template file. If incoming image template does not exist use current image template file. """ - def mount_next_image_fs(squashfs_path, mount_point): - run_command_or_raise(["mkdir", "-p", mount_point]) - run_command_or_raise(["mount", "-t", "squashfs", squashfs_path, mount_point]) - - def umount_next_image_fs(mount_point): - run_command_or_raise(["umount", "-rf", mount_point]) - run_command_or_raise(["rm", "-rf", mount_point]) SONIC_ENV_TEMPLATE_FILE = os.path.join("usr", "share", "sonic", "templates", "sonic-environment.j2") SONIC_VERSION_YML_FILE = os.path.join("etc", "sonic", "sonic_version.yml") @@ -236,7 +267,7 @@ def umount_next_image_fs(mount_point): env_file = os.path.join(env_dir, "sonic-environment") try: - mount_next_image_fs(new_image_squashfs_path, new_image_mount) + mount_squash_fs(new_image_squashfs_path, new_image_mount) next_sonic_env_template_file = os.path.join(new_image_mount, SONIC_ENV_TEMPLATE_FILE) next_sonic_version_yml_file = os.path.join(new_image_mount, SONIC_VERSION_YML_FILE) @@ -260,7 +291,46 @@ def umount_next_image_fs(mount_point): os.remove(env_file) os.rmdir(env_dir) finally: - umount_next_image_fs(new_image_mount) + umount(new_image_mount) + + +def migrate_sonic_packages(image_version): + """ Migrate SONiC packages to new SONiC image. """ + + SONIC_PACKAGE_MANAGER = "sonic-package-manager" + PACKAGE_MANAGER_DIR = "/var/lib/sonic-package-manager/" + DOCKER_CTL_SCRIPT = "/usr/lib/docker/docker.sh" + + sonic_version = re.sub("SONiC-OS-", '', image_version) + new_image_dir = os.path.join('/', "host", "image-{0}".format(sonic_version)) + new_image_squashfs_path = os.path.join(new_image_dir, "fs.squashfs") + new_image_upper_dir = os.path.join(new_image_dir, "rw") + new_image_work_dir = os.path.join(new_image_dir, "work") + new_image_docker_dir = os.path.join(new_image_dir, "docker") + new_image_mount = os.path.join('/', "tmp", "image-{0}-fs".format(sonic_version)) + new_image_docker_mount = os.path.join(new_image_mount, "var", "lib", "docker") + + try: + packages_file = "packages.json" + packages_path = os.path.join(PACKAGE_MANAGER_DIR, packages_file) + tmp_dir = "/tmp" + mount_squash_fs(new_image_squashfs_path, new_image_mount) + # make sure upper dir and work dir exist + run_command_or_raise(["mkdir", "-p", new_image_upper_dir]) + run_command_or_raise(["mkdir", "-p", new_image_work_dir]) + mount_overlay_fs(new_image_mount, new_image_upper_dir, new_image_work_dir, new_image_mount) + mount_bind(new_image_docker_dir, new_image_docker_mount) + mount_procfs_chroot(new_image_mount) + mount_sysfs_chroot(new_image_mount) + run_command_or_raise(["chroot", new_image_mount, DOCKER_CTL_SCRIPT, "start"]) + run_command_or_raise(["cp", packages_path, os.path.join(new_image_mount, tmp_dir, packages_file)]) + run_command_or_raise(["chroot", new_image_mount, SONIC_PACKAGE_MANAGER, "migrate", + "{}".format(os.path.join(tmp_dir, packages_file)), "-y"]) + finally: + run_command("chroot {} {} stop".format(new_image_mount, DOCKER_CTL_SCRIPT)) + umount(new_image_mount, recursive=True, read_only=False, remove_dir=False) + umount(new_image_mount) + # Main entrypoint @click.group(cls=AliasedGroup) @@ -282,8 +352,10 @@ def sonic_installer(): help="Force installation of an image of a type which differs from that of the current running image") @click.option('--skip_migration', is_flag=True, help="Do not migrate current configuration to the newly installed image") +@click.option('--skip-package-migration', is_flag=True, + help="Do not migrate current packages to the newly installed image") @click.argument('url') -def install(url, force, skip_migration=False): +def install(url, force, skip_migration=False, skip_package_migration=False): """ Install image from local binary or URL""" bootloader = get_bootloader() @@ -329,6 +401,9 @@ def install(url, force, skip_migration=False): update_sonic_environment(click, binary_image_version) + if not skip_package_migration: + migrate_sonic_packages(binary_image_version) + # Finally, sync filesystem run_command("sync;sync;sync") run_command("sleep 3") # wait 3 seconds after sync From ece38450f94a49bca688e06f1d92f8592e616ccf Mon Sep 17 00:00:00 2001 From: Stepan Blyshchak Date: Tue, 3 Nov 2020 16:36:41 +0200 Subject: [PATCH 002/173] fix temp dir path for chroot Signed-off-by: Stepan Blyshchak --- sonic_installer/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sonic_installer/main.py b/sonic_installer/main.py index 43da10f11f..3930e6d59a 100644 --- a/sonic_installer/main.py +++ b/sonic_installer/main.py @@ -313,7 +313,7 @@ def migrate_sonic_packages(image_version): try: packages_file = "packages.json" packages_path = os.path.join(PACKAGE_MANAGER_DIR, packages_file) - tmp_dir = "/tmp" + tmp_dir = "tmp" mount_squash_fs(new_image_squashfs_path, new_image_mount) # make sure upper dir and work dir exist run_command_or_raise(["mkdir", "-p", new_image_upper_dir]) @@ -325,7 +325,7 @@ def migrate_sonic_packages(image_version): run_command_or_raise(["chroot", new_image_mount, DOCKER_CTL_SCRIPT, "start"]) run_command_or_raise(["cp", packages_path, os.path.join(new_image_mount, tmp_dir, packages_file)]) run_command_or_raise(["chroot", new_image_mount, SONIC_PACKAGE_MANAGER, "migrate", - "{}".format(os.path.join(tmp_dir, packages_file)), "-y"]) + "{}".format(os.path.join("/", tmp_dir, packages_file)), "-y"]) finally: run_command("chroot {} {} stop".format(new_image_mount, DOCKER_CTL_SCRIPT)) umount(new_image_mount, recursive=True, read_only=False, remove_dir=False) From 8bc75a769e3a4cd3b1a12876e189e412c9b70cca Mon Sep 17 00:00:00 2001 From: Stepan Blyshchak Date: Tue, 3 Nov 2020 19:45:01 +0200 Subject: [PATCH 003/173] fix for install of already installed image Signed-off-by: Stepan Blyshchak --- sonic_installer/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sonic_installer/main.py b/sonic_installer/main.py index 3930e6d59a..919854fdaa 100644 --- a/sonic_installer/main.py +++ b/sonic_installer/main.py @@ -401,8 +401,8 @@ def install(url, force, skip_migration=False, skip_package_migration=False): update_sonic_environment(click, binary_image_version) - if not skip_package_migration: - migrate_sonic_packages(binary_image_version) + if not skip_package_migration: + migrate_sonic_packages(binary_image_version) # Finally, sync filesystem run_command("sync;sync;sync") From e72f54059bb8538b993a8a06f0e03031582b255f Mon Sep 17 00:00:00 2001 From: Stepan Blyshchak Date: Tue, 15 Dec 2020 19:14:39 +0200 Subject: [PATCH 004/173] move constants to the top Signed-off-by: Stepan Blyshchak --- sonic_installer/main.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sonic_installer/main.py b/sonic_installer/main.py index 919854fdaa..661419ead8 100644 --- a/sonic_installer/main.py +++ b/sonic_installer/main.py @@ -301,19 +301,19 @@ def migrate_sonic_packages(image_version): PACKAGE_MANAGER_DIR = "/var/lib/sonic-package-manager/" DOCKER_CTL_SCRIPT = "/usr/lib/docker/docker.sh" + tmp_dir = "tmp" + packages_file = "packages.json" + packages_path = os.path.join(PACKAGE_MANAGER_DIR, packages_file) sonic_version = re.sub("SONiC-OS-", '', image_version) new_image_dir = os.path.join('/', "host", "image-{0}".format(sonic_version)) new_image_squashfs_path = os.path.join(new_image_dir, "fs.squashfs") new_image_upper_dir = os.path.join(new_image_dir, "rw") new_image_work_dir = os.path.join(new_image_dir, "work") new_image_docker_dir = os.path.join(new_image_dir, "docker") - new_image_mount = os.path.join('/', "tmp", "image-{0}-fs".format(sonic_version)) + new_image_mount = os.path.join('/', tmp_dir, "image-{0}-fs".format(sonic_version)) new_image_docker_mount = os.path.join(new_image_mount, "var", "lib", "docker") try: - packages_file = "packages.json" - packages_path = os.path.join(PACKAGE_MANAGER_DIR, packages_file) - tmp_dir = "tmp" mount_squash_fs(new_image_squashfs_path, new_image_mount) # make sure upper dir and work dir exist run_command_or_raise(["mkdir", "-p", new_image_upper_dir]) From be4f089e635494328a21f01c671da50efd6bc80d Mon Sep 17 00:00:00 2001 From: Stepan Blyshchak Date: Thu, 21 Jan 2021 15:36:22 +0200 Subject: [PATCH 005/173] [sonic_installer] migrate packages from old docker library Signed-off-by: Stepan Blyshchak --- sonic_installer/main.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/sonic_installer/main.py b/sonic_installer/main.py index 9de42f3a3d..e26d188600 100644 --- a/sonic_installer/main.py +++ b/sonic_installer/main.py @@ -294,6 +294,8 @@ def migrate_sonic_packages(image_version): SONIC_PACKAGE_MANAGER = "sonic-package-manager" PACKAGE_MANAGER_DIR = "/var/lib/sonic-package-manager/" DOCKER_CTL_SCRIPT = "/usr/lib/docker/docker.sh" + DOCKERD_SOCK = "docker.sock" + VAR_RUN_PATH = "/var/run/" tmp_dir = "tmp" packages_file = "packages.json" @@ -318,8 +320,14 @@ def migrate_sonic_packages(image_version): mount_sysfs_chroot(new_image_mount) run_command_or_raise(["chroot", new_image_mount, DOCKER_CTL_SCRIPT, "start"]) run_command_or_raise(["cp", packages_path, os.path.join(new_image_mount, tmp_dir, packages_file)]) + run_command_or_raise(["touch", os.path.join(new_image_mount, "tmp", DOCKERD_SOCK)]) + run_command_or_raise(["mount", "--bind", + os.path.join(VAR_RUN_PATH, DOCKERD_SOCK), + os.path.join(new_image_mount, "tmp", DOCKERD_SOCK)]) run_command_or_raise(["chroot", new_image_mount, SONIC_PACKAGE_MANAGER, "migrate", - "{}".format(os.path.join("/", tmp_dir, packages_file)), "-y"]) + os.path.join("/", tmp_dir, packages_file), + "--dockerd-socket", os.path.join("/", tmp_dir, DOCKERD_SOCK), + "-y"]) finally: run_command("chroot {} {} stop".format(new_image_mount, DOCKER_CTL_SCRIPT)) umount(new_image_mount, recursive=True, read_only=False, remove_dir=False) From 9fd283d3656cf0741b43d70be6b88992bf3cf51d Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Thu, 18 Mar 2021 15:58:55 +0200 Subject: [PATCH 006/173] remove unused imports Signed-off-by: Stepan Blyschak --- sonic_installer/bootloader/aboot.py | 1 - sonic_installer/bootloader/bootloader.py | 1 - 2 files changed, 2 deletions(-) diff --git a/sonic_installer/bootloader/aboot.py b/sonic_installer/bootloader/aboot.py index 3dbb29c2f7..ef1ea622cd 100644 --- a/sonic_installer/bootloader/aboot.py +++ b/sonic_installer/bootloader/aboot.py @@ -19,7 +19,6 @@ HOST_PATH, IMAGE_DIR_PREFIX, IMAGE_PREFIX, - ROOTFS_NAME, run_command, run_command_or_raise, ) diff --git a/sonic_installer/bootloader/bootloader.py b/sonic_installer/bootloader/bootloader.py index fba5c79f49..8ce2041f0a 100644 --- a/sonic_installer/bootloader/bootloader.py +++ b/sonic_installer/bootloader/bootloader.py @@ -9,7 +9,6 @@ HOST_PATH, IMAGE_DIR_PREFIX, IMAGE_PREFIX, - ROOTFS_NAME, ) class Bootloader(object): From 0cbaa3e2feced78f843f91c6f999ca7d96a22360 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Fri, 19 Mar 2021 11:27:45 +0200 Subject: [PATCH 007/173] fix redefining path Signed-off-by: Stepan Blyschak --- sonic_installer/bootloader/bootloader.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sonic_installer/bootloader/bootloader.py b/sonic_installer/bootloader/bootloader.py index 8ce2041f0a..607cd730db 100644 --- a/sonic_installer/bootloader/bootloader.py +++ b/sonic_installer/bootloader/bootloader.py @@ -70,6 +70,6 @@ def get_image_path(cls, image): return image.replace(IMAGE_PREFIX, prefix) @contextmanager - def get_path_in_image(self, image_path, path): + def get_path_in_image(self, image_path, path_in_image): """returns the path to the squashfs""" - yield path.join(image_path, path) \ No newline at end of file + yield path.join(image_path, path_in_image) \ No newline at end of file From 1a9979737f1a4980a60cac3c1620e155812d2185 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Tue, 23 Mar 2021 10:21:26 +0000 Subject: [PATCH 008/173] [sonic-package-manager] add new utility to manage SONiC packages Signed-off-by: Stepan Blyschak --- setup.py | 15 + .../bash_completion.d/sonic-package-manager | 8 + sonic-utilities-data/bash_completion.d/spm | 8 + sonic-utilities-data/templates/dump.sh.j2 | 29 + sonic-utilities-data/templates/monit.conf.j2 | 18 + .../templates/service_mgmt.sh.j2 | 149 +++ .../templates/sonic.service.j2 | 39 + sonic-utilities-data/templates/timer.unit.j2 | 15 + sonic_package_manager/__init__.py | 5 + sonic_package_manager/constraint.py | 140 +++ sonic_package_manager/database.py | 222 +++++ sonic_package_manager/dockerapi.py | 226 +++++ sonic_package_manager/errors.py | 146 +++ sonic_package_manager/logger.py | 29 + sonic_package_manager/main.py | 453 +++++++++ sonic_package_manager/manager.py | 896 ++++++++++++++++++ sonic_package_manager/manifest.py | 210 ++++ sonic_package_manager/metadata.py | 185 ++++ sonic_package_manager/package.py | 53 ++ sonic_package_manager/progress.py | 52 + sonic_package_manager/reference.py | 30 + sonic_package_manager/registry.py | 156 +++ .../service_creator/__init__.py | 3 + .../service_creator/creator.py | 339 +++++++ .../service_creator/feature.py | 108 +++ .../service_creator/sonic_db.py | 97 ++ .../service_creator/utils.py | 17 + sonic_package_manager/source.py | 178 ++++ sonic_package_manager/utils.py | 42 + sonic_package_manager/version.py | 23 + tests/sonic_package_manager/conftest.py | 379 ++++++++ tests/sonic_package_manager/test_cli.py | 63 ++ .../sonic_package_manager/test_constraint.py | 75 ++ tests/sonic_package_manager/test_database.py | 89 ++ tests/sonic_package_manager/test_manager.py | 321 +++++++ tests/sonic_package_manager/test_manifest.py | 67 ++ tests/sonic_package_manager/test_metadata.py | 36 + tests/sonic_package_manager/test_reference.py | 17 + tests/sonic_package_manager/test_registry.py | 15 + .../test_service_creator.py | 173 ++++ tests/sonic_package_manager/test_utils.py | 8 + 41 files changed, 5134 insertions(+) create mode 100644 sonic-utilities-data/bash_completion.d/sonic-package-manager create mode 100644 sonic-utilities-data/bash_completion.d/spm create mode 100644 sonic-utilities-data/templates/dump.sh.j2 create mode 100644 sonic-utilities-data/templates/monit.conf.j2 create mode 100644 sonic-utilities-data/templates/service_mgmt.sh.j2 create mode 100644 sonic-utilities-data/templates/sonic.service.j2 create mode 100644 sonic-utilities-data/templates/timer.unit.j2 create mode 100644 sonic_package_manager/__init__.py create mode 100644 sonic_package_manager/constraint.py create mode 100644 sonic_package_manager/database.py create mode 100644 sonic_package_manager/dockerapi.py create mode 100644 sonic_package_manager/errors.py create mode 100644 sonic_package_manager/logger.py create mode 100644 sonic_package_manager/main.py create mode 100644 sonic_package_manager/manager.py create mode 100644 sonic_package_manager/manifest.py create mode 100644 sonic_package_manager/metadata.py create mode 100644 sonic_package_manager/package.py create mode 100644 sonic_package_manager/progress.py create mode 100644 sonic_package_manager/reference.py create mode 100644 sonic_package_manager/registry.py create mode 100644 sonic_package_manager/service_creator/__init__.py create mode 100644 sonic_package_manager/service_creator/creator.py create mode 100644 sonic_package_manager/service_creator/feature.py create mode 100644 sonic_package_manager/service_creator/sonic_db.py create mode 100644 sonic_package_manager/service_creator/utils.py create mode 100644 sonic_package_manager/source.py create mode 100644 sonic_package_manager/utils.py create mode 100644 sonic_package_manager/version.py create mode 100644 tests/sonic_package_manager/conftest.py create mode 100644 tests/sonic_package_manager/test_cli.py create mode 100644 tests/sonic_package_manager/test_constraint.py create mode 100644 tests/sonic_package_manager/test_database.py create mode 100644 tests/sonic_package_manager/test_manager.py create mode 100644 tests/sonic_package_manager/test_manifest.py create mode 100644 tests/sonic_package_manager/test_metadata.py create mode 100644 tests/sonic_package_manager/test_reference.py create mode 100644 tests/sonic_package_manager/test_registry.py create mode 100644 tests/sonic_package_manager/test_service_creator.py create mode 100644 tests/sonic_package_manager/test_utils.py diff --git a/setup.py b/setup.py index 8018efd82c..9847435671 100644 --- a/setup.py +++ b/setup.py @@ -44,6 +44,8 @@ 'show.interfaces', 'sonic_installer', 'sonic_installer.bootloader', + 'sonic_package_manager', + 'sonic_package_manager.service_creator', 'tests', 'undebug', 'utilities_common', @@ -146,19 +148,29 @@ 'sonic-clear = clear.main:cli', 'sonic-installer = sonic_installer.main:sonic_installer', 'sonic_installer = sonic_installer.main:sonic_installer', # Deprecated + 'sonic-package-manager = sonic_package_manager.main:cli', + 'spm = sonic_package_manager.main:cli', 'undebug = undebug.main:cli', 'watchdogutil = watchdogutil.main:watchdogutil', ] }, install_requires=[ 'click==7.0', + 'click-log==0.3.2', + 'docker==4.4.4', + 'docker-image-py==0.1.10', + 'filelock==3.0.12', + 'enlighten==1.8.0', 'ipaddress==1.0.23', + 'jinja2==2.11.3', 'jsondiff==1.2.0', 'm2crypto==0.31.0', 'natsort==6.2.1', # 6.2.1 is the last version which supports Python 2. Can update once we no longer support Python 2 'netaddr==0.8.0', 'netifaces==0.10.7', 'pexpect==4.8.0', + 'poetry-semver==0.1.0', + 'prettyprinter==0.18.0', 'pyroute2==0.5.14', 'requests==2.25.0', 'sonic-platform-common', @@ -166,6 +178,7 @@ 'sonic-yang-mgmt', 'swsssdk>=2.0.1', 'tabulate==0.8.2', + 'www-authenticate==0.9.2', 'xmltodict==0.12.0', ], setup_requires= [ @@ -173,7 +186,9 @@ 'wheel' ], tests_require = [ + 'pyfakefs', 'pytest', + 'mock>=2.0.0', 'mockredispy>=2.9.3', 'sonic-config-engine' ], diff --git a/sonic-utilities-data/bash_completion.d/sonic-package-manager b/sonic-utilities-data/bash_completion.d/sonic-package-manager new file mode 100644 index 0000000000..a8a2456603 --- /dev/null +++ b/sonic-utilities-data/bash_completion.d/sonic-package-manager @@ -0,0 +1,8 @@ +_sonic_package_manager_completion() { + COMPREPLY=( $( env COMP_WORDS="${COMP_WORDS[*]}" \ + COMP_CWORD=$COMP_CWORD \ + _SONIC_PACKAGE_MANAGER_COMPLETE=complete $1 ) ) + return 0 +} + +complete -F _sonic_package_manager_completion -o default sonic-package-manager; diff --git a/sonic-utilities-data/bash_completion.d/spm b/sonic-utilities-data/bash_completion.d/spm new file mode 100644 index 0000000000..8931dc389c --- /dev/null +++ b/sonic-utilities-data/bash_completion.d/spm @@ -0,0 +1,8 @@ +_sonic_package_manager_completion() { + COMPREPLY=( $( env COMP_WORDS="${COMP_WORDS[*]}" \ + COMP_CWORD=$COMP_CWORD \ + _SONIC_PACKAGE_MANAGER_COMPLETE=complete $1 ) ) + return 0 +} + +complete -F _sonic_package_manager_completion -o default spm; diff --git a/sonic-utilities-data/templates/dump.sh.j2 b/sonic-utilities-data/templates/dump.sh.j2 new file mode 100644 index 0000000000..ebb7ed8f24 --- /dev/null +++ b/sonic-utilities-data/templates/dump.sh.j2 @@ -0,0 +1,29 @@ +#!/bin/bash + +# +# =============== Managed by SONiC Package Manager. DO NOT EDIT! =============== +# auto-generated from {{ source }} by sonic-package-manager +# + +service="{{ manifest.service.name }}" +dump_command="{{ manifest.package['debug-dump'] }}" +container_re="^${service}[0-9]*$" +{% raw %} +container_ids="$(docker ps -f name=${container_re} -f status=running --format {{.Names}})" +{% endraw %} +tmp_dir=$(mktemp -d) +tmp_dump_dir="$tmp_dir/$service" +tmp_archive=$(mktemp) + +mkdir -p "$tmp_dump_dir" + +for container_id in $container_ids; do + docker exec -t "${container_id}" ${dump_command} &> "${tmp_dump_dir}/${container_id}" +done + + +tar -C $(dirname $tmp_dump_dir) -cf $tmp_archive $service + +cat $tmp_archive +rm $tmp_archive +rm -rf $tmp_dir diff --git a/sonic-utilities-data/templates/monit.conf.j2 b/sonic-utilities-data/templates/monit.conf.j2 new file mode 100644 index 0000000000..f51efb9bee --- /dev/null +++ b/sonic-utilities-data/templates/monit.conf.j2 @@ -0,0 +1,18 @@ +############################################################################### +## +## =============== Managed by SONiC Package Manager. DO NOT EDIT! =============== +## auto-generated from {{ source }} by sonic-package-manager +## +## Monit configuration for {{ feature }} service +## process list: +{%- for process in processes %} +{%- if process.critical %} +## {{ process.name }} +{%- endif %} +{%- endfor %} +############################################################################### +{%- for process in processes %} +check program {{ feature }}|{{ process.name }} with path "/usr/bin/process_checker {{ feature }} {{ process.command }}" + if status != 0 for 5 times within 5 cycles then alert + +{% endfor %} diff --git a/sonic-utilities-data/templates/service_mgmt.sh.j2 b/sonic-utilities-data/templates/service_mgmt.sh.j2 new file mode 100644 index 0000000000..e46ba47380 --- /dev/null +++ b/sonic-utilities-data/templates/service_mgmt.sh.j2 @@ -0,0 +1,149 @@ +#!/bin/bash + +# +# =============== Managed by SONiC Package Manager. DO NOT EDIT! =============== +# auto-generated from {{ source }} by sonic-package-manager +# + +SERVICE="{{ manifest.service.name }}" +NAMESPACE_PREFIX="asic" +SONIC_DB_CLI="sonic-db-cli" +TMPDIR="/tmp/" +DEBUGLOG="${TMPDIR}/${SERVICE}.log" +[[ ! -z $DEV ]] && DEBUGLOG="${TMPDIR}/${SERVICE}-${DEV}.log" +[[ ! -z $DEV ]] && NET_NS="${NAMESPACE_PREFIX}${DEV}" # name of the network namespace +[[ ! -z $DEV ]] && SONIC_DB_CLI="${SONIC_DB_CLI} -n ${NET_NS}" + +{%- for service in manifest.service.dependent %} +{%- if service in multi_instance_services %} +MULTI_INST_DEPENDENT="${MULTI_INST_DEPENDENT} {{ service }}" +{%- else %} +DEPENDENT="${DEPENDENT} {{ service }}" +{%- endif %} +{%- endfor %} + +# Update dependent list based on other packages requirements +if [[ -f /etc/sonic/${SERVICE}_dependent ]]; then + DEPENDENT="${DEPENDENT} $(cat /etc/sonic/${SERVICE}_dependent)" +fi + +if [[ -f /etc/sonic/${SERVICE}_multi_inst_dependent ]]; then + MULTI_INST_DEPENDENT="${MULTI_INST_DEPENDENT} cat /etc/sonic/${SERVICE}_multi_inst_dependent" +fi + +function debug() +{ + /usr/bin/logger $1 + /bin/echo `date` "- $1" >> ${DEBUGLOG} +} + +function check_warm_boot() +{ + SYSTEM_WARM_START=`$SONIC_DB_CLI STATE_DB hget "WARM_RESTART_ENABLE_TABLE|system" enable` + SERVICE_WARM_START=`$SONIC_DB_CLI STATE_DB hget "WARM_RESTART_ENABLE_TABLE|${SERVICE}" enable` + if [[ x"$SYSTEM_WARM_START" == x"true" ]] || [[ x"$SERVICE_WARM_START" == x"true" ]]; then + WARM_BOOT="true" +{#- TODO: restore count validation for SONiC packages #} + else + WARM_BOOT="false" + fi +} + +function check_fast_boot() +{ + if [[ $($SONIC_DB_CLI STATE_DB GET "FAST_REBOOT|system") == "1" ]]; then + FAST_BOOT="true" + else + FAST_BOOT="false" + fi +} + +function start_dependent_services() { + if [[ x"$WARM_BOOT" != x"true" ]]; then + for dep in ${DEPENDENT}; do + /bin/systemctl start ${dep} + done + for dep in ${MULTI_INST_DEPENDENT}; do + if [[ ! -z $DEV ]]; then + /bin/systemctl start ${dep}@$DEV + else + /bin/systemctl start ${dep} + fi + done + fi +} + +function stop_dependent_services() { + if [[ x"$WARM_BOOT" != x"true" ]] && [[ x"$FAST_BOOT" != x"true" ]]; then + for dep in ${DEPENDENT}; do + /bin/systemctl stop ${dep} + done + for dep in ${MULTI_INST_DEPENDENT}; do + if [[ ! -z $DEV ]]; then + /bin/systemctl stop ${dep}@$DEV + else + /bin/systemctl stop ${dep} + fi + done + fi +} + +function start() { + debug "Starting ${SERVICE}$DEV service..." + + # start service docker + /usr/bin/${SERVICE}.sh start $DEV + debug "Started ${SERVICE}$DEV service..." + +{%- if manifest.service["post-start-action"] %} + docker exec -t ${SERVICE}${DEV} {{ manifest.service["post-start-action"] }} +{%- endif %} +} + +function wait() { + start_dependent_services + + if [[ ! -z $DEV ]]; then + /usr/bin/${SERVICE}.sh wait $DEV + else + /usr/bin/${SERVICE}.sh wait + fi +} + +function stop() { + debug "Stopping ${SERVICE}$DEV service..." + +{%- if manifest.service["pre-shutdown-action"] %} + docker exec -t ${SERVICE}${DEV} {{ manifest.service["pre-shutdown-action"] }} +{%- endif %} + + # For WARM/FAST boot do not perform service stop + if [[ x"$WARM_BOOT" != x"true" ]] && [[ x"$FAST_BOOT" != x"true" ]]; then + /usr/bin/${SERVICE}.sh stop $DEV + else + docker kill ${SERVICE}$DEV &> /dev/null || debug "Docker ${SERVICE}$DEV is not running ($?) ..." + fi + + debug "Stopped ${SERVICE}$DEV service..." + + stop_dependent_services +} + +OP=$1 +DEV=$2 + +check_warm_boot +check_fast_boot + +debug "Fast boot flag: ${SERVICE}$DEV ${FAST_BOOT}." +debug "Warm boot flag: ${SERVICE}$DEV ${WARM_BOOT}." + +case "$OP" in + start|wait|stop) + $1 + ;; + *) + echo "Usage: $0 {start|wait|stop}" + exit 1 + ;; +esac diff --git a/sonic-utilities-data/templates/sonic.service.j2 b/sonic-utilities-data/templates/sonic.service.j2 new file mode 100644 index 0000000000..72d6ab698c --- /dev/null +++ b/sonic-utilities-data/templates/sonic.service.j2 @@ -0,0 +1,39 @@ +# +# =============== Managed by SONiC Package Manager. DO NOT EDIT! =============== +# auto-generated from {{ source }} by sonic-package-manager +# +{%- set path = '/usr/local/bin' %} +{%- set multi_instance = multi_instance|default(False) %} +{%- set multi_instance_services = multi_instance_services|default([]) %} +[Unit] +Description={{ manifest.service.name }} container +{%- for service in manifest.service.requires %} +Requires={{ service }}{% if multi_instance and service in multi_instance_services %}@%i{% endif %}.service +{%- endfor %} +{%- for service in manifest.service.requisite %} +Requisite={{ service }}{% if multi_instance and service in multi_instance_services %}@%i{% endif %}.service +{%- endfor %} +{%- for service in manifest.service.after %} +After={{ service }}{% if multi_instance and service in multi_instance_services %}@%i{% endif %}.service +{%- endfor %} +{%- for service in manifest.service.before %} +Before={{ service }}{% if multi_instance and service in multi_instance_services %}@%i{% endif %}.service +{%- endfor %} +BindsTo=sonic.target +After=sonic.target +StartLimitIntervalSec=1200 +StartLimitBurst=3 + +[Service] +ExecStartPre={{path}}/{{manifest.service.name}}.sh start{% if multi_instance %} %i{% endif %} +ExecStart={{path}}/{{manifest.service.name}}.sh wait{% if multi_instance %} %i{% endif %} +ExecStop={{path}}/{{manifest.service.name}}.sh stop{% if multi_instance %} %i{% endif %} +RestartSec=30 + +{%- if not manifest.service.delayed %} +[Install] +WantedBy=sonic.target +{%- for service in manifest.service["wanted-by"] %} +WantedBy={{ service }}{% if multi_instance and service in multi_instance_services %}@%i{% endif %}.service +{%- endfor %} +{%- endif %} diff --git a/sonic-utilities-data/templates/timer.unit.j2 b/sonic-utilities-data/templates/timer.unit.j2 new file mode 100644 index 0000000000..a757b8deb8 --- /dev/null +++ b/sonic-utilities-data/templates/timer.unit.j2 @@ -0,0 +1,15 @@ +# +# =============== Managed by SONiC Package Manager. DO NOT EDIT! =============== +# auto-generated from {{ source }} by sonic-package-manager +# +[Unit] +Description=Delays {{ manifest.service.name }} until SONiC has started +PartOf={{ manifest.service.name }}{% if multi_instance %}@%i{% endif %}.service + +[Timer] +OnUnitActiveSec=0 sec +OnBootSec=3min 30 sec +Unit={{ manifest.service.name }}{% if multi_instance %}@%i{% endif %}.service + +[Install] +WantedBy=timers.target sonic.target diff --git a/sonic_package_manager/__init__.py b/sonic_package_manager/__init__.py new file mode 100644 index 0000000000..9d8827c5e4 --- /dev/null +++ b/sonic_package_manager/__init__.py @@ -0,0 +1,5 @@ +#!/usr/bin/env python + +from sonic_package_manager.manager import PackageManager + +__all__ = ['PackageManager'] diff --git a/sonic_package_manager/constraint.py b/sonic_package_manager/constraint.py new file mode 100644 index 0000000000..09f0fbc0fe --- /dev/null +++ b/sonic_package_manager/constraint.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python + +""" Package version constraints module. """ + +import re +from abc import ABC +from dataclasses import dataclass, field +from typing import Dict, Union + +import semver + + +class VersionConstraint(semver.VersionConstraint, ABC): + """ Extends VersionConstraint from semver package. """ + + @staticmethod + def parse(constraint_expression: str) -> 'VersionConstraint': + """ Parse version constraint. + + Args: + constraint_expression: Expression syntax: "[[op][version]]+". + Returns: + The resulting VersionConstraint object. + """ + + return semver.parse_constraint(constraint_expression) + + +@dataclass +class ComponentConstraints: + """ ComponentConstraints is a set of components version constraints. """ + + components: Dict[str, VersionConstraint] = field(default_factory=dict) + + @staticmethod + def parse(constraints: Dict) -> 'ComponentConstraints': + """ Parse constraint from dictionary. + + Args: + constraints: dictionary with component name + as key and constraint expression as value + + Returns: + ComponentConstraints object. + + """ + + components = {component: VersionConstraint.parse(version) + for component, version in constraints.items()} + return ComponentConstraints(components) + + +@dataclass +class PackageConstraint: + """ PackageConstraint is a package version constraint. """ + + name: str + constraint: VersionConstraint + components: Dict[str, VersionConstraint] = field(default_factory=dict) + + def __str__(self): + return f'{self.name}{self.constraint}' + + @staticmethod + def from_string(constraint_expression: str) -> 'PackageConstraint': + """ Parse package constraint string which contains a package + name separated by a space with zero, one or more version constraint + expressions. A variety of version matching operators are supported + including >, <, ==, !=, ^, *. See Examples. + + Args: + constraint_expression: Expression syntax "[package name] [[op][version]]+". + + Returns: + PackageConstraint object. + + Examples: + >>> PackageConstraint.parse('syncd^1.0.0').constraint + =1.0.0,<2.0.0)> + >>> PackageConstraint.parse('swss>1.3.2 <4.2.1').constraint + 1.3.2,<4.2.1)> + >>> PackageConstraint.parse('swss').constraint + + """ + + REQUIREMENT_SPECIFIER_RE = \ + r'(?P[A-Za-z0-9_-]+)(?P.*)' + + match = re.match(REQUIREMENT_SPECIFIER_RE, constraint_expression) + if match is None: + raise ValueError(f'Invalid constraint {constraint_expression}') + groupdict = match.groupdict() + name = groupdict.get('name') + constraint = groupdict.get('constraint') or '*' + return PackageConstraint(name, VersionConstraint.parse(constraint)) + + @staticmethod + def from_dict(constraint_dict: Dict) -> 'PackageConstraint': + """ Parse package constraint information from dictionary. E.g: + + { + "name": "swss", + "version": "^1.0.0", + "componenets": { + "libswsscommon": "^1.0.0" + } + } + + Args: + constraint_dict: Dictionary of constraint infromation. + + Returns: + PackageConstraint object. + """ + + name = constraint_dict['name'] + version = VersionConstraint.parse(constraint_dict.get('version') or '*') + components = {component: VersionConstraint.parse(version) + for component, version in constraint_dict.get('components', {}).items()} + return PackageConstraint(name, version, components) + + @staticmethod + def parse(constraint: Union[str, Dict]) -> 'PackageConstraint': + """ Parse constraint from string expression or dictionary. + + Args: + constraint: string or dictionary. Check from_str() and from_dict() methods. + + Returns: + PackageConstraint object. + + """ + + if type(constraint) is str: + return PackageConstraint.from_string(constraint) + elif type(constraint) is dict: + return PackageConstraint.from_dict(constraint) + else: + raise ValueError('Input argument should be either str or dict') + diff --git a/sonic_package_manager/database.py b/sonic_package_manager/database.py new file mode 100644 index 0000000000..6c1cec5c07 --- /dev/null +++ b/sonic_package_manager/database.py @@ -0,0 +1,222 @@ +#!/usr/bin/env python + +""" Repository Database interface module. """ +import json +import os +from dataclasses import dataclass, replace +from typing import Optional, Dict, Callable + +from sonic_package_manager.errors import PackageManagerError, PackageNotFoundError, PackageAlreadyExistsError +from sonic_package_manager.version import Version + +BASE_LIBRARY_PATH = '/var/lib/sonic-package-manager/' +PACKAGE_MANAGER_DB_FILE_PATH = os.path.join(BASE_LIBRARY_PATH, 'packages.json') +PACKAGE_MANAGER_LOCK_FILE = os.path.join(BASE_LIBRARY_PATH, '.lock') + + +@dataclass(order=True) +class PackageEntry: + """ Package database single entry object. + + Attributes: + name: Name of the package + repository: Default repository to pull package from. + description: Package description or None if package does not + provide a description. + default_reference: Default reference (tag or digest) or None + if default reference is not provided. + version: Installed version of the package or None if + package is not installed. + installed: Boolean flag whether the package is installed. + built_in: Boolean flag whether the package is built in. + image_id: Image ID for this package or None if package + is not installed. + """ + + name: str + repository: Optional[str] + description: Optional[str] = None + default_reference: Optional[str] = None + version: Optional[Version] = None + installed: bool = False + built_in: bool = False + image_id: Optional[str] = None + + +def package_from_dict(name: str, package_info: Dict) -> PackageEntry: + """ Parse dictionary into PackageEntry object.""" + + repository = package_info.get('repository') + description = package_info.get('description') + default_reference = package_info.get('default-reference') + version = package_info.get('installed-version') + if version: + version = Version.parse(version) + installed = package_info.get('installed', False) + built_in = package_info.get('built-in', False) + image_id = package_info.get('image-id') + + return PackageEntry(name, repository, description, + default_reference, version, installed, + built_in, image_id) + + +def package_to_dict(package: PackageEntry) -> Dict: + """ Serialize package into dictionary. """ + + return { + 'repository': package.repository, + 'description': package.description, + 'default-reference': package.default_reference, + 'installed-version': None if package.version is None else str(package.version), + 'installed': package.installed, + 'built-in': package.built_in, + 'image-id': package.image_id, + } + + +class PackageDatabase: + """ An interface to SONiC repository database """ + + def __init__(self, + database: Dict[str, PackageEntry], + on_save: Optional[Callable] = None): + """ Initialize PackageDatabase. + + Args: + database: Database dictionary + on_save: Optional callback to execute on commit() + """ + + self._database = database + self._on_save = on_save + + def add_package(self, + name: str, + repository: str, + description: Optional[str] = None, + default_reference: Optional[str] = None): + """ Adds a new package entry in database. + + Args: + name: Package name. + repository: Repository URL. + description: Description string. + default_reference: Default version string. + + Raises: + PackageAlreadyExistsError: if package already exists in database. + """ + + if self.has_package(name): + raise PackageAlreadyExistsError(name) + + package = PackageEntry(name, repository, description, default_reference) + self._database[name] = package + + def remove_package(self, name: str): + """ Removes package entry from database. + + Args: + name: repository name. + Raises: + PackageNotFoundError: Raises when package with the given name does not exist + in the database. + """ + + pkg = self.get_package(name) + + if pkg.built_in: + raise PackageManagerError(f'Package {name} is built-in, cannot remove it') + + if pkg.installed: + raise PackageManagerError(f'Package {name} is installed, uninstall it first') + + self._database.pop(name) + + def update_package(self, pkg: PackageEntry): + """ Modify repository in the database. + + Args: + pkg: Repository object. + Raises: + PackageManagerError: Raises when repository with the given name does not exist + in the database. + """ + + name = pkg.name + + if not self.has_package(name): + raise PackageNotFoundError(name) + + self._database[name] = pkg + + def get_package(self, name: str) -> PackageEntry: + """ Return a package referenced by name. + If the package is not found PackageNotFoundError is thrown. + + Args: + name: Package name. + Returns: + PackageInfo object. + Raises: + PackageNotFoundError: When package called name was not found. + """ + + try: + pkg = self._database[name] + except KeyError: + raise PackageNotFoundError(name) + + return replace(pkg) + + def has_package(self, name: str) -> bool: + """ Checks if the database contains an entry for a package. + called name. Returns True if the package exists, otherwise False. + + Args: + name: Package name. + Returns: + True if the package exists, otherwise False. + """ + + try: + self.get_package(name) + return True + except PackageNotFoundError: + return False + + def __iter__(self): + """ Iterates over packages in the database. + + Yields: + PackageInfo object. + """ + + for name, _ in self._database.items(): + yield self.get_package(name) + + @staticmethod + def from_file(db_file=PACKAGE_MANAGER_DB_FILE_PATH) -> 'PackageDatabase': + """ Read database content from file. """ + + def on_save(database): + with open(db_file, 'w') as db: + db_content = {} + for name, package in database.items(): + db_content[name] = package_to_dict(package) + json.dump(db_content, db, indent=4) + + database = {} + with open(db_file) as db: + db_content = json.load(db) + for key in db_content: + package = package_from_dict(key, db_content[key]) + database[key] = package + return PackageDatabase(database, on_save) + + def commit(self): + """ Save database content to file. """ + + if self._on_save: + self._on_save(self._database) diff --git a/sonic_package_manager/dockerapi.py b/sonic_package_manager/dockerapi.py new file mode 100644 index 0000000000..926600d0bc --- /dev/null +++ b/sonic_package_manager/dockerapi.py @@ -0,0 +1,226 @@ +#!/usr/bin/evn python + +""" Module provides Docker interface. """ + +import contextlib +import io +import tarfile +import re +from typing import Optional + +from sonic_package_manager.logger import log +from sonic_package_manager.progress import ProgressManager + + +def is_digest(ref: str): + return ref.startswith('sha256:') + + +def bytes_to_mb(bytes): + return bytes / 1024 / 1024 + + +def get_id(line): + return line['id'] + + +def get_status(line): + return line['status'] + + +def get_progress(line): + progress = line['progressDetail'] + current = bytes_to_mb(progress['current']) + total = bytes_to_mb(progress['total']) + return current, total + + +def process_progress(progress_manager, line): + try: + status = get_status(line) + id = get_id(line) + current, total = get_progress(line) + + if id not in progress_manager: + progress_manager.new(id, + total=total, + unit='Mb', + desc=f'{status} {id}') + pbar = progress_manager.get(id) + + # Complete status + if 'complete' in status: + pbar.desc = f'{status} {id}' + pbar.update(pbar.total) + return + + # Status changed + if status not in pbar.desc: + pbar.desc = f'{status} {id}' + pbar.total = total + pbar.count = 0 + + pbar.update(current - pbar.count) + except KeyError: + # not a progress line + return + + +def get_repository_from_image(image): + """ Returns the first RepoTag repository + found in image. """ + + repotags = image.attrs['RepoTags'] + for repotag in repotags: + repository, tag = repotag.split(':') + return repository + + +class DockerApi: + """ DockerApi provides a set of methods - + wrappers around docker client methods """ + + def __init__(self, + client, + progress_manager: Optional[ProgressManager] = None): + self.client = client + self.progress_manager = progress_manager + + def pull(self, repository: str, + reference: Optional[str] = None): + """ Docker 'pull' command. + Args: + repository: repository to pull + reference: tag or digest + """ + + log.debug(f'pulling image from {repository} reference={reference}') + + api = self.client.api + progress_manager = self.progress_manager + + digest = None + + with progress_manager or contextlib.nullcontext(): + for line in api.pull(repository, + reference, + stream=True, + decode=True): + log.debug(f'pull status: {line}') + + status = get_status(line) + + # Record pulled digest + digest_match = re.match(r'Digest: (?P.*)', status) + if digest_match: + digest = digest_match.groupdict()['sha'] + + if progress_manager: + process_progress(progress_manager, line) + + log.debug(f'Digest: {digest}') + log.debug(f'image from {repository} reference={reference} pulled successfully') + + return self.get_image(f'{repository}@{digest}') + + def load(self, imgpath: str): + """ Docker 'load' command. + Args: + + """ + + log.debug(f'loading image from {imgpath}') + + api = self.client.api + progress_manager = self.progress_manager + + imageid = None + repotag = None + + with progress_manager or contextlib.nullcontext(): + with open(imgpath, 'rb') as imagefile: + for line in api.load_image(imagefile, quiet=False): + log.debug(f'pull status: {line}') + + if progress_manager: + process_progress(progress_manager, line) + + if 'stream' not in line: + continue + + stream = line['stream'] + repotag_match = re.match(r'Loaded image: (?P.*)\n', stream) + if repotag_match: + repotag = repotag_match.groupdict()['repotag'] + imageid_match = re.match(r'Loaded image ID: sha256:(?P.*)\n', stream) + if imageid_match: + imageid = imageid_match.groupdict()['id'] + + imagename = repotag if repotag else imageid + log.debug(f'Loaded image {imagename}') + + return self.get_image(imagename) + + def rmi(self, image: str, **kwargs): + """ Docker 'rmi -f' command. """ + + log.debug(f'removing image {image} kwargs={kwargs}') + + self.client.images.remove(image, **kwargs) + + log.debug(f'image {image} removed successfully') + + def tag(self, image: str, repotag: str, **kwargs): + """ Docker 'tag' command """ + + log.debug(f'tagging image {image} {repotag} kwargs={kwargs}') + + img = self.client.images.get(image) + img.tag(repotag, **kwargs) + + log.debug(f'image {image} tagged {repotag} successfully') + + def rm(self, container: str, **kwargs): + """ Docker 'rm' command. """ + + self.client.containers.get(container).remove(**kwargs) + log.debug(f'removed container {container}') + + def ps(self, **kwargs): + """ Docker 'ps' command. """ + + return self.client.containers.list(**kwargs) + + def labels(self, image: str): + """ Returns a list of labels associated with image. """ + + log.debug(f'inspecting image labels {image}') + + labels = self.client.images.get(image).labels + + log.debug(f'image {image} labels successfully: {labels}') + return labels + + def get_image(self, name: str): + return self.client.images.get(name) + + def extract(self, image, src_path: str, dst_path: str): + """ Copy src_path from the docker image to host dst_path. """ + + buf = bytes() + + container = self.client.containers.create(image) + try: + bits, _ = container.get_archive(src_path) + for chunk in bits: + buf += chunk + finally: + container.remove(force=True) + + with tarfile.open(fileobj=io.BytesIO(buf)) as tar: + for member in tar: + if dst_path.endswith('/'): + tar.extract(member, dst_path) + else: + member.name = dst_path + tar.extract(member, dst_path) diff --git a/sonic_package_manager/errors.py b/sonic_package_manager/errors.py new file mode 100644 index 0000000000..17279c52c4 --- /dev/null +++ b/sonic_package_manager/errors.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python + +""" SONiC Package Manager exceptions are defined in this module. """ + +from dataclasses import dataclass +from typing import Optional + +from sonic_package_manager.constraint import PackageConstraint, VersionConstraint +from sonic_package_manager.version import Version + + +class PackageManagerError(Exception): + """ Base class for exceptions generated by SONiC package manager """ + + pass + + +class ManifestError(Exception): + """ Class for manifest validate failures. """ + + pass + + +class MetadataError(Exception): + """ Class for metadata failures. """ + + pass + + +@dataclass +class PackageNotFoundError(PackageManagerError): + """ Repository not found in repository database exception """ + + name: str + + def __str__(self): + return f'Package {self.name} is not found in packages database' + + +@dataclass +class PackageAlreadyExistsError(PackageManagerError): + """ Package already exists in the packages database exception. """ + + name: str + + def __str__(self): + return f'Package {self.name} already exists in packages database' + + +class PackageInstallationError(PackageManagerError): + """ Exception for package installation error. """ + + pass + + +class PackageUninstallationError(PackageManagerError): + """ Exception for package installation error. """ + + pass + + +class PackageUpgradeError(PackageManagerError): + """ Exception for package upgrade error. """ + + pass + + +@dataclass +class PackageSonicRequirementError(PackageInstallationError): + """ Exception for installation errors, when SONiC version requirement is not met. """ + + name: str + component: str + constraint: PackageConstraint + installed_ver: Optional[Version] = None + + def __str__(self): + if self.installed_ver is not None: + return (f'Package {self.name} requires base OS component {self.component} version {self.constraint} ' + f'while the installed version is {self.installed_ver}') + return (f'Package {self.name} requires base OS component {self.component} version {self.constraint} ' + f'but it is not present int base OS image') + + +@dataclass +class PackageDependencyError(PackageInstallationError): + """ Exception class for installation errors related to missing dependency. """ + + name: str + constraint: PackageConstraint + installed_ver: Optional[Version] = None + + def __str__(self): + if self.installed_ver: + return (f'Package {self.name} requires {self.constraint} ' + f'but version {self.installed_ver} is installed') + return f'Package {self.name} requires {self.constraint} but it is not installed' + + +@dataclass +class PackageComponentDependencyError(PackageInstallationError): + """ Exception class for installation error caused by component + version dependency. """ + + name: str + dependency: str + component: str + constraint: VersionConstraint + installed_ver: Optional[Version] = None + + def __str__(self): + if self.installed_ver: + return (f'Package {self.name} requires {self.component} {self.constraint} ' + f'in package {self.dependency} but version {self.installed_ver} is installed') + return (f'Package {self.name} requires {self.component} {self.constraint} ' + f'in package {self.dependency} but it is not installed') + + +@dataclass +class PackageConflictError(PackageInstallationError): + """ Exception class for installation errors related to missing dependency. """ + + name: str + constraint: PackageConstraint + installed_ver: Version + + def __str__(self): + return (f'Package {self.name} conflicts with {self.constraint} but ' + f'version {self.installed_ver} is installed') + + +@dataclass +class PackageComponentConflictError(PackageInstallationError): + """ Exception class for installation error caused by component + version conflict. """ + + name: str + dependency: str + component: str + constraint: VersionConstraint + installed_ver: Version + + def __str__(self): + return (f'Package {self.name} conflicts with {self.component} {self.constraint} ' + f'in package {self.dependency} but version {self.installed_ver} is installed') + diff --git a/sonic_package_manager/logger.py b/sonic_package_manager/logger.py new file mode 100644 index 0000000000..3d5e06d35f --- /dev/null +++ b/sonic_package_manager/logger.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python + +""" Logger for sonic-package-manager. """ + +import logging.handlers + +import click_log + + +class Formatter(click_log.ColorFormatter): + """ Click logging formatter. """ + + colors = { + 'error': dict(fg='red'), + 'exception': dict(fg='red'), + 'critical': dict(fg='red'), + 'debug': dict(fg='blue', bold=True), + 'warning': dict(fg='yellow'), + } + + +log = logging.getLogger("sonic-package-manager") +log.setLevel(logging.INFO) + +click_handler = click_log.ClickHandler() +click_handler.formatter = Formatter() + +log.addHandler(click_handler) +log.addHandler(logging.handlers.SysLogHandler()) diff --git a/sonic_package_manager/main.py b/sonic_package_manager/main.py new file mode 100644 index 0000000000..dcc048079c --- /dev/null +++ b/sonic_package_manager/main.py @@ -0,0 +1,453 @@ +#!/usr/bin/env python + +import functools +import json +import os +import sys +import typing + +import click +import click_log +import tabulate +from natsort import natsorted + +from sonic_package_manager.database import PackageEntry, PackageDatabase +from sonic_package_manager.errors import PackageManagerError +from sonic_package_manager.logger import log +from sonic_package_manager.manager import PackageManager + +BULLET_UC = '\u2022' + + +def exit_cli(*args, **kwargs): + """ Print a message and exit with rc 1. """ + + click.secho(*args, **kwargs) + sys.exit(1) + + +def show_help(ctx): + """ Show help message and exit process successfully. """ + + click.echo(ctx.get_help()) + ctx.exit(0) + + +def root_privileges_required(func: typing.Callable) -> typing.Callable: + """ Decorates a function, so that the function is invoked + only if the user is root. """ + + @functools.wraps(func) + def wrapped_function(*args, **kwargs): + """ Wrapper around func. """ + + if os.geteuid() != 0: + exit_cli('Root privileges required for this operation', fg='red') + + return func(*args, **kwargs) + + return wrapped_function + + +def add_options(options): + """ Decorator to append options from + input list to command. """ + + def _add_options(func): + for option in reversed(options): + func = option(func) + return func + + return _add_options + + +class MutuallyExclusiveOption(click.Option): + """ This options type is extended with 'mutually_exclusive' + parameter which makes CLI to check if several options are now + used together in single command. """ + + def __init__(self, *args, **kwargs): + self.mutually_exclusive = set(kwargs.pop('mutually_exclusive', [])) + help_string = kwargs.get('help', '') + if self.mutually_exclusive: + ex_str = ', '.join(self.mutually_exclusive) + kwargs['help'] = f'{help_string} ' \ + f'NOTE: This argument is mutually ' \ + f'exclusive with arguments: [{ex_str}].' + super().__init__(*args, **kwargs) + + def handle_parse_result(self, ctx, opts, args): + if self.name in opts and opts[self.name] is not None: + for opt_name in self.mutually_exclusive.intersection(opts): + if opts[opt_name] is None: + continue + + raise click.UsageError(f'Illegal usage: {self.name} is mutually ' + f'exclusive with arguments ' + f'{", ".join(self.mutually_exclusive)}.') + + return super().handle_parse_result(ctx, opts, args) + + +PACKAGE_SOURCE_OPTIONS = [ + click.option('--from-repository', + help='Install package directly from image registry repository', + cls=MutuallyExclusiveOption, + mutually_exclusive=['from_tarball', 'package_expr']), + click.option('--from-tarball', + type=click.Path(exists=True, + readable=True, + file_okay=True, + dir_okay=False), + help='Install package from saved image tarball', + cls=MutuallyExclusiveOption, + mutually_exclusive=['from_repository', 'package_expr']), + click.argument('package-expr', + type=str, + required=False) +] + + +PACKAGE_COMMON_INSTALL_OPTIONS = [ + click.option('--skip-cli-plugin-installation', + is_flag=True, + help='Do not install CLI plugins provided by the package ' + 'on the host OS. NOTE: In case when package /cli/mandatory ' + 'field is set to True this option will fail the installation.'), +] + + +PACKAGE_COMMON_OPERATION_OPTIONS = [ + click.option('-f', '--force', + is_flag=True, + help='Force operation by ignoring failures'), + click.option('-y', '--yes', + is_flag=True, + help='Automatically answer yes on prompts'), + click_log.simple_verbosity_option(log), +] + + +def get_package_status(package: PackageEntry): + """ Returns the installation status message for package. """ + + if package.built_in: + return 'Built-In' + elif package.installed: + return 'Installed' + else: + return 'Not Installed' + + +@click.group() +@click.pass_context +def cli(ctx): + """ SONiC Package Manager """ + + ctx.obj = PackageManager.get_manager() + + +@cli.group() +@click.pass_context +def repository(ctx): + """ Repository management commands """ + + pass + + +@cli.group() +@click.pass_context +def show(ctx): + """ Package manager show commands """ + + pass + + +@show.group() +@click.pass_context +def package(ctx): + """ Package show commands """ + + pass + + +@cli.command() +@click.pass_context +def list(ctx): + """ List available repositories """ + + table_header = ['Name', 'Repository', 'Description', 'Version', 'Status'] + table_body = [] + + manager: PackageManager = ctx.obj + + try: + for package in natsorted(manager.database): + repository = package.repository or 'N/A' + version = package.version or 'N/A' + description = package.description or 'N/A' + status = get_package_status(package) + + table_body.append([ + package.name, + repository, + description, + version, + status + ]) + + click.echo(tabulate.tabulate(table_body, table_header)) + except PackageManagerError as err: + exit_cli(f'Failed to list repositories: {err}', fg='red') + + +@package.command() +@add_options(PACKAGE_SOURCE_OPTIONS) +@click.pass_context +def manifest(ctx, + package_expr, + from_repository, + from_tarball): + """ Print package manifest content """ + + manager: PackageManager = ctx.obj + + try: + source = manager.get_package_source(package_expr, + from_repository, + from_tarball) + package = source.get_package() + click.echo(json.dumps(package.manifest.unmarshal(), indent=4)) + except Exception as err: + exit_cli(f'Failed to print manifest: {err}', fg='red') + + +@package.command() +@click.argument('name') +@click.option('--all', is_flag=True, help='Show all available tags in repository') +@click.option('--plain', is_flag=True, help='Plain output') +@click.pass_context +def versions(ctx, name, all, plain): + """ Print available versions """ + + try: + manager: PackageManager = ctx.obj + versions = manager.get_package_available_versions(name, all) + for version in versions: + if not plain: + click.secho(f'{BULLET_UC} ', bold=True, fg='green', nl=False) + click.secho(f'{version}') + except Exception as err: + exit_cli(f'Failed to get package versions for {name}: {err}', fg='red') + + +@package.command() +@add_options(PACKAGE_SOURCE_OPTIONS) +@click.pass_context +def changelog(ctx, + package_expr, + from_repository, + from_tarball): + """ Print package changelog """ + + manager: PackageManager = ctx.obj + + try: + source = manager.get_package_source(package_expr, + from_repository, + from_tarball) + package = source.get_package() + changelog = package.manifest['package']['changelog'] + + if not changelog: + raise PackageManagerError(f'No changelog for package {package.name}') + + for version, entry in changelog.items(): + author = entry.get('author') or 'N/A' + email = entry.get('email') or 'N/A' + changes = entry.get('changes') or [] + date = entry.get('date') or 'N/A' + click.secho(f'{version}:\n', fg='green', bold=True) + for line in changes: + click.secho(f' {BULLET_UC} {line}', bold=True) + click.secho(f'\n {author} ' + f'({email}) {date}', fg='green', bold=True) + click.secho('') + + except Exception as err: + exit_cli(f'Failed to print package changelog: {err}', fg='red') + + +@repository.command() +@click.argument('name', type=str) +@click.argument('repository', type=str) +@click.option('--default-reference', type=str) +@click.option('--description', type=str) +@click.pass_context +@root_privileges_required +def add(ctx, name, repository, default_reference, description): + """ Add a new repository to database. """ + + manager: PackageManager = ctx.obj + + try: + manager.add_repository(name, + repository, + description=description, + default_reference=default_reference) + except Exception as err: + exit_cli(f'Failed to add repository {name}: {err}', fg='red') + + +@repository.command() +@click.argument("name") +@click.pass_context +@root_privileges_required +def remove(ctx, name): + """ Remove package from database. """ + + manager: PackageManager = ctx.obj + + try: + manager.remove_repository(name) + except Exception as err: + exit_cli(f'Failed to remove repository {name}: {err}', fg='red') + + +@cli.command() +@click.option('--enable', + is_flag=True, + help='Set the default state of the feature to enabled ' + 'and enable feature right after installation. ' + 'NOTE: user needs to execute "config save -y" to make ' + 'this setting persistent') +@click.option('--default-owner', + type=click.Choice(['local', 'kube']), + default='local', + help='Default owner configuration setting for a feature') +@add_options(PACKAGE_SOURCE_OPTIONS) +@add_options(PACKAGE_COMMON_OPERATION_OPTIONS) +@add_options(PACKAGE_COMMON_INSTALL_OPTIONS) +@click.pass_context +@root_privileges_required +def install(ctx, + package_expr, + from_repository, + from_tarball, + force, + yes, + enable, + default_owner, + skip_cli_plugin_installation): + """ Install package """ + + manager: PackageManager = ctx.obj + + package_source = package_expr or from_repository or from_tarball + + if not yes and not force: + click.confirm(f'{package_source} is going to be installed, ' + f'continue?', abort=True, show_default=True) + + install_opts = { + 'force': force, + 'enable': enable, + 'default_owner': default_owner, + 'skip_cli_plugin_installation': skip_cli_plugin_installation, + } + + try: + manager.install(package_expr, + from_repository, + from_tarball, + **install_opts) + except Exception as err: + exit_cli(f'Failed to install {package_source}: {err}', fg='red') + except KeyboardInterrupt: + exit_cli(f'Operation canceled by user', fg='red') + + +@cli.command() +@add_options(PACKAGE_SOURCE_OPTIONS) +@add_options(PACKAGE_COMMON_OPERATION_OPTIONS) +@add_options(PACKAGE_COMMON_INSTALL_OPTIONS) +@click.pass_context +@root_privileges_required +def upgrade(ctx, + package_expr, + from_repository, + from_tarball, + force, + yes): + """ Upgrade package """ + + manager: PackageManager = ctx.obj + + package_source = package_expr or from_repository or from_tarball + + if not yes and not force: + click.confirm(f'Package is going to be upgraded with {package_source}, ' + f'continue?', abort=True, show_default=True) + + upgrade_opts = { + 'force': force, + 'skip_cli_plugin_installation': skip_cli_plugin_installation, + } + + try: + manager.upgrade(package_expr, + from_repository, + from_tarball, + **upgrade_opts) + except Exception as err: + exit_cli(f'Failed to upgrade {package_source}: {err}', fg='red') + except KeyboardInterrupt: + exit_cli(f'Operation canceled by user', fg='red') + + +@cli.command() +@add_options(PACKAGE_COMMON_OPERATION_OPTIONS) +@click.argument('name') +@click.pass_context +@root_privileges_required +def uninstall(ctx, name, force, yes): + """ Uninstall package """ + + manager: PackageManager = ctx.obj + + if not yes and not force: + click.confirm(f'Package {name} is going to be uninstalled, ' + f'continue?', abort=True, show_default=True) + + try: + manager.uninstall(name, force) + except Exception as err: + exit_cli(f'Failed to uninstall package {name}: {err}', fg='red') + except KeyboardInterrupt: + exit_cli(f'Operation canceled by user', fg='red') + + +@cli.command() +@add_options(PACKAGE_COMMON_OPERATION_OPTIONS) +@click.option('--dockerd-socket', type=click.Path()) +@click.argument('database', type=click.Path()) +@click.pass_context +@root_privileges_required +def migrate(ctx, database, force, yes, dockerd_socket): + """ Migrate packages from the given database file """ + + manager: PackageManager = ctx.obj + + if not yes and not force: + click.confirm('Continue with package migration?', abort=True, show_default=True) + + try: + manager.migrate_packages(PackageDatabase.from_file(database), dockerd_socket) + except Exception as err: + exit_cli(f'Failed to migrate packages {err}', fg='red') + except KeyboardInterrupt: + exit_cli(f'Operation canceled by user', fg='red') + + +if __name__ == "__main__": + cli() diff --git a/sonic_package_manager/manager.py b/sonic_package_manager/manager.py new file mode 100644 index 0000000000..f1643a1846 --- /dev/null +++ b/sonic_package_manager/manager.py @@ -0,0 +1,896 @@ +#!/usr/bin/env python +import contextlib +import functools +import os +import pkgutil +import tempfile +from typing import Any, Iterable, Callable, Dict, Optional + +import docker +import filelock +from sonic_py_common import device_info + +from sonic_package_manager import utils +from sonic_package_manager.constraint import ( + VersionConstraint, + PackageConstraint +) +from sonic_package_manager.database import ( + PACKAGE_MANAGER_LOCK_FILE, + PackageDatabase +) +from sonic_package_manager.dockerapi import DockerApi +from sonic_package_manager.errors import ( + PackageManagerError, + PackageDependencyError, + PackageComponentDependencyError, + PackageConflictError, + PackageComponentConflictError, + PackageInstallationError, + PackageSonicRequirementError, + PackageUninstallationError, + PackageUpgradeError +) +from sonic_package_manager.logger import log +from sonic_package_manager.metadata import MetadataResolver +from sonic_package_manager.package import Package +from sonic_package_manager.progress import ProgressManager +from sonic_package_manager.reference import PackageReference +from sonic_package_manager.registry import RegistryResolver +from sonic_package_manager.service_creator.creator import ServiceCreator, run_command +from sonic_package_manager.service_creator.feature import FeatureRegistry +from sonic_package_manager.service_creator.sonic_db import SonicDB +from sonic_package_manager.service_creator.utils import in_chroot +from sonic_package_manager.source import ( + PackageSource, + LocalSource, + RegistrySource, + TarballSource +) +from sonic_package_manager.utils import DockerReference +from sonic_package_manager.version import ( + Version, + VersionRange, + version_to_tag, tag_to_version +) + + +@contextlib.contextmanager +def failure_ignore(ignore: bool): + """ Ignores failures based on parameter passed. """ + + try: + yield + except Exception as err: + if ignore: + log.warning(f'ignoring error {err}') + else: + raise + + +def under_lock(func: Callable) -> Callable: + """ Execute operations under lock. """ + + @functools.wraps(func) + def wrapped_function(*args, **kwargs): + self = args[0] + with self.lock: + return func(*args, **kwargs) + + return wrapped_function + + +def rollback_wrapper(func, *args, **kwargs): + """ Used in rollback callbacks to ignore failure + but proceed with rollback. Error will be printed + but not fail the whole procedure of rollback. """ + + @functools.wraps(func) + def wrapper(): + try: + func(*args, **kwargs) + except Exception as err: + log.error(f'failed in rollback: {err}') + + return wrapper + + +def package_constraint_to_reference(constraint: PackageConstraint) -> PackageReference: + package_name, version_constraint = constraint.name, constraint.constraint + # Allow only specific version for now. + # Later we can improve package manager to support + # installing packages using expressions like 'package>1.0.0' + if version_constraint == VersionRange(): # empty range means any version + return PackageReference(package_name, None) + if not isinstance(version_constraint, Version): + raise PackageManagerError(f'Can only install specific version. ' + f'Use only following expression "{package_name}==" ' + f'to install specific version') + return PackageReference(package_name, version_to_tag(version_constraint)) + + +def parse_reference_expression(expression): + try: + return package_constraint_to_reference(PackageConstraint.parse(expression)) + except ValueError: + # if we failed to parse the expression as constraint expression + # we will try to parse it as reference + return PackageReference.parse(expression) + + +def validate_package_base_os_constraints(package: Package, sonic_version_info: Dict[str, str]): + """ Verify that all dependencies on base OS components are met. + Args: + package: Package to check constraints for. + sonic_version_info: SONiC components version information. + Raises: + PackageSonicRequirementError: in case dependency is not satisfied. + """ + + base_os_constraints = package.manifest['package']['base-os'].components + for component, constraint in base_os_constraints.items(): + if component not in sonic_version_info: + raise PackageSonicRequirementError(package.name, component, constraint) + + version = Version.parse(sonic_version_info[component]) + + if not constraint.allows_all(version): + raise PackageSonicRequirementError(package.name, component, constraint, version) + + +def validate_package_tree(packages: Dict[str, Package]): + """ Verify that all dependencies are met in all packages passed to this function. + Args: + packages: list of packages to check + Raises: + PackageDependencyError: if dependency is missing + PackageConflictError: if there is a conflict between packages + """ + + for name, package in packages.items(): + log.debug(f'checking dependencies for {name}') + for dependency in package.manifest['package']['depends']: + dependency_package = packages.get(dependency.name) + if dependency_package is None: + raise PackageDependencyError(package.name, dependency) + + installed_version = dependency_package.version + log.debug(f'dependency package is installed {dependency.name}: {installed_version}') + if not dependency.constraint.allows_all(installed_version): + raise PackageDependencyError(package.name, dependency, installed_version) + + dependency_components = dependency.components + if not dependency_components: + dependency_components = {} + for component, version in package.components.items(): + implicit_constraint = VersionConstraint.parse(f'^{version.major}.{version.minor}.0') + dependency_components[component] = implicit_constraint + + for component, constraint in dependency_components.items(): + if component not in dependency_package.components: + raise PackageComponentDependencyError(package.name, dependency, + component, constraint) + + component_version = dependency_package.components[component] + log.debug(f'dependency package {dependency.name}: ' + f'component {component} version is {component_version}') + + if not constraint.allows_all(component_version): + raise PackageComponentDependencyError(package.name, dependency, component, + constraint, component_version) + + log.debug(f'checking conflicts for {name}') + for conflict in package.manifest['package']['breaks']: + conflicting_package = packages.get(conflict.name) + if conflicting_package is None: + continue + + installed_version = conflicting_package.version + log.debug(f'conflicting package is installed {conflict.name}: {installed_version}') + if conflict.constraint.allows_all(installed_version): + raise PackageConflictError(package.name, conflict, installed_version) + + for component, constraint in conflicting_package.components.items(): + if component not in conflicting_package.components: + continue + + component_version = conflicting_package.components[component] + log.debug(f'conflicting package {dependency.name}: ' + f'component {component} version is {component_version}') + + if constraint.allows_all(component_version): + raise PackageComponentConflictError(package.name, dependency, component, + constraint, component_version) + + +def validate_package_cli_can_be_skipped(package: Package, skip: bool): + """ Checks whether package CLI installation can be skipped. + + Args: + package: Package to validate + skip: Whether to skip installing CLI + + Raises: + PackageManagerError + + """ + + if package.manifest['cli']['mandatory'] and skip: + raise PackageManagerError(f'CLI is mandatory for package {package.name} ' + f'but it was requested to be not installed') + elif skip: + log.warning(f'Package {package.name} CLI plugin will not be installed') + + +class PackageManager: + """ SONiC Package Manager. This class provides public API + for sonic_package_manager python library. It has functionality + for installing, uninstalling, updating SONiC packages as well as + retrieving information about the packages from different sources. """ + + def __init__(self, + docker_api: DockerApi, + registry_resolver: RegistryResolver, + database: PackageDatabase, + metadata_resolver: MetadataResolver, + service_creator: ServiceCreator, + device_information: Any, + lock: filelock.FileLock): + """ Initialize PackageManager. """ + + self.lock = lock + self.docker = docker_api + self.registry_resolver = registry_resolver + self.database = database + self.metadata_resolver = metadata_resolver + self.service_creator = service_creator + self.feature_registry = service_creator.feature_registry + self.is_multi_npu = device_information.is_multi_npu() + self.num_npus = device_information.get_num_npus() + self.version_info = device_information.get_sonic_version_info() + + @under_lock + def add_repository(self, *args, **kwargs): + """ Add repository to package database + and commit database content. + + Args: + args: Arguments to pass to PackageDatabase.add_package + kwargs: Keyword arguments to pass to PackageDatabase.add_package + """ + + self.database.add_package(*args, **kwargs) + self.database.commit() + + @under_lock + def remove_repository(self, name: str): + """ Remove repository from package database + and commit database content. + + Args: + name: package name + """ + + self.database.remove_package(name) + self.database.commit() + + @under_lock + def install(self, + expression: Optional[str] = None, + repotag: Optional[str] = None, + tarball: Optional[str] = None, + **kwargs): + """ Install SONiC Package from either an expression representing + the package and its version, repository and tag or digest in + same format as "docker pulL" accepts or an image tarball path. + + Args: + expression: SONiC Package reference expression + repotag: Install from REPO[:TAG][@DIGEST] + tarball: Install from tarball, path to tarball file + kwargs: Install options for self.install_from_source + Raises: + PackageManagerError + """ + + source = self.get_package_source(expression, repotag, tarball) + self.install_from_source(source, **kwargs) + + @under_lock + def install_from_source(self, + source: PackageSource, + force=False, + enable=False, + default_owner='local', + skip_cli_plugin_installation=False): + """ Install SONiC Package from source represented by PackageSource. + This method contains the logic of package installation. + + Args: + source: SONiC Package source. + force: Force the installation. + enable: If True the installed feature package will be enabled. + default_owner: Owner of the installed package. + skip_cli_plugin_installation: Skip CLI plugin installation. + Raises: + PackageManagerError + """ + + package = source.get_package() + name = package.name + + with failure_ignore(force): + if self.is_installed(name): + raise PackageInstallationError(f'{name} is already installed') + + version = package.manifest['package']['version'] + feature_state = 'enabled' if enable else 'disabled' + installed_packages = self._get_installed_packages_and(package) + + with failure_ignore(force): + validate_package_base_os_constraints(package, self.version_info) + validate_package_tree(installed_packages) + validate_package_cli_can_be_skipped(package, skip_cli_plugin_installation) + + # After all checks are passed we proceed to actual installation + + # When installing package from a tarball or directly from registry + # package name may not be in database. + if not self.database.has_package(package.name): + self.database.add_package(package.name, package.repository) + + try: + with contextlib.ExitStack() as exit_stack: + source.install(package) + exit_stack.callback(rollback_wrapper(source.uninstall, package)) + + self.service_creator.create(package, state=feature_state, owner=default_owner) + exit_stack.callback(rollback_wrapper(self.service_creator.remove, package)) + + if not skip_cli_plugin_installation: + self._install_cli_plugins(package) + exit_stack.callback(rollback_wrapper(self._uninstall_cli_plugins, package)) + + exit_stack.pop_all() + except Exception as err: + raise PackageInstallationError(f'Failed to install {package.name}: {err}') + except KeyboardInterrupt: + raise + + package.entry.installed = True + package.entry.version = version + self.database.update_package(package.entry) + self.database.commit() + + @under_lock + def uninstall(self, name: str, force=False): + """ Uninstall SONiC Package referenced by name. The uninstallation + can be forced if force argument is True. + + Args: + name: SONiC Package name. + force: Force the installation. + Raises: + PackageManagerError + """ + + with failure_ignore(force): + if not self.is_installed(name): + raise PackageUninstallationError(f'{name} is not installed') + + package = self.get_installed_package(name) + service_name = package.manifest['service']['name'] + + with failure_ignore(force): + if self.feature_registry.is_feature_enabled(service_name): + raise PackageUninstallationError( + f'{service_name} is enabled. Disable the feature first') + + if package.built_in: + raise PackageUninstallationError( + f'Cannot uninstall built-in package {package.name}') + + installed_packages = self._get_installed_packages_except(package) + + with failure_ignore(force): + validate_package_tree(installed_packages) + + # After all checks are passed we proceed to actual uninstallation + + try: + self._uninstall_cli_plugins(package) + self.service_creator.remove(package) + + # Clean containers based on this image + containers = self.docker.ps(filters={'ancestor': package.image_id}, all=True) + for container in containers: + self.docker.rm(container.id, force=True) + + self.docker.rmi(package.image_id, force=True) + package.entry.image_id = None + except Exception as err: + raise PackageUninstallationError(f'Failed to uninstall {package.name}: {err}') + + package.entry.installed = False + package.entry.version = None + self.database.update_package(package.entry) + self.database.commit() + + @under_lock + def upgrade(self, + expression: Optional[str] = None, + repotag: Optional[str] = None, + tarball: Optional[str] = None, + **kwargs): + """ Upgrade SONiC Package from either an expression representing + the package and its version, repository and tag or digest in + same format as "docker pulL" accepts or an image tarball path. + + Args: + expression: SONiC Package reference expression + repotag: Upgrade from REPO[:TAG][@DIGEST] + tarball: Upgrade from tarball, path to tarball file + kwargs: Upgrade options for self.upgrade_from_source + Raises: + PackageManagerError + """ + + source = self.get_package_source(expression, repotag, tarball) + self.upgrade_from_source(source, **kwargs) + + @under_lock + def upgrade_from_source(self, + source: PackageSource, + force=False, + skip_cli_plugin_installation=False): + """ Upgrade SONiC Package to a version the package reference + expression specifies. Can force the upgrade if force parameter + is True. Force can allow a package downgrade. + + Args: + source: SONiC Package source + force: Force the upgrade. + skip_cli_plugin_installation: Skip CLI plugin installation. + Raises: + PackageManagerError + """ + + new_package = source.get_package() + name = new_package.name + + with failure_ignore(force): + if not self.is_installed(name): + raise PackageUpgradeError(f'{name} is not installed') + + old_package = self.get_installed_package(name) + + if old_package.built_in: + raise PackageUpgradeError(f'Cannot upgrade built-in package {old_package.name}') + + old_feature = old_package.manifest['service']['name'] + new_feature = new_package.manifest['service']['name'] + old_version = old_package.manifest['package']['version'] + new_version = new_package.manifest['package']['version'] + + with failure_ignore(force): + if old_version == new_version: + raise PackageUpgradeError(f'{new_version} is already installed') + + # TODO: Not all packages might support downgrade. + # We put a check here but we understand that for some packages + # the downgrade might be safe to do. In that case we might want to + # add another argument to this function: allow_downgrade: bool = False. + # Another way to do that might be a variable in manifest describing package + # downgrade ability or downgrade-able versions. + if new_version < old_version: + raise PackageUpgradeError(f'Request to downgrade from {old_version} to {new_version}. ' + f'Downgrade might be not supported by the package') + + # remove currently installed package from the list + installed_packages = self._get_installed_packages_and(new_package) + + with failure_ignore(force): + validate_package_base_os_constraints(new_package, self.version_info) + validate_package_tree(installed_packages) + validate_package_cli_can_be_skipped(new_package, skip_cli_plugin_installation) + + # After all checks are passed we proceed to actual upgrade + + try: + with contextlib.ExitStack() as exit_stack: + self._uninstall_cli_plugins(old_package) + exit_stack.callback(rollback_wrapper(self._install_cli_plugins, old_package)) + + source.install(new_package) + exit_stack.callback(rollback_wrapper(source.uninstall, new_package)) + + if self.feature_registry.is_feature_enabled(old_feature): + self._systemctl_action(old_package, 'stop') + exit_stack.callback(rollback_wrapper(self._systemctl_action, + old_package, 'start')) + + self.service_creator.remove(old_package, deregister_feature=False) + exit_stack.callback(rollback_wrapper(self.service_creator.create, + old_package, register_feature=False)) + + # This is no return point, after we start removing old Docker images + # there is no guaranty we can actually successfully roll-back. + + # Clean containers based on the old image + containers = self.docker.ps(filters={'ancestor': old_package.image_id}, all=True) + for container in containers: + self.docker.rm(container.id, force=True) + + self.docker.rmi(old_package.image_id, force=True) + + self.service_creator.create(new_package, register_feature=False) + + if self.feature_registry.is_feature_enabled(new_feature): + self._systemctl_action(new_package, 'start') + + if not skip_cli_plugin_installation: + self._install_cli_plugins(new_package) + + exit_stack.pop_all() + except Exception as err: + raise PackageUpgradeError(f'Failed to upgrade {new_package.name}: {err}') + except KeyboardInterrupt: + raise + + new_package_entry = new_package.entry + new_package_entry.installed = True + new_package_entry.version = new_version + self.database.update_package(new_package_entry) + self.database.commit() + + @under_lock + def migrate_packages(self, + old_package_database: PackageDatabase, + dockerd_sock: Optional[str] = None): + """ Migrate packages from old database. This function can + do a comparison between current database and the database + passed in as argument. + If the package is missing in the current database it will be added. + If the package is installed in the passed database and in the current + it is not installed it will be installed with a passed database package version. + If the package is installed in the passed database and it is installed + in the current database but with older version the package will be upgraded to + the never version. + If the package is installed in the passed database and in the current + it is installed but with never version - no actions are taken. + If dockerd_sock parameter is passed, the migration process will use loaded + images from docker library of the currently installed image. + + Args: + old_package_database: SONiC Package Database to migrate packages from. + dockerd_sock: Path to dockerd socket. + Raises: + PackageManagerError + """ + + self._migrate_package_database(old_package_database) + + def migrate_package(old_package_entry, + new_package_entry, + migrate_operation=None): + """ Migrate package routine + + Args: + old_package_entry: Entry in old package database. + new_package_entry: Entry in new package database. + migrate_operation: Operation to perform: install or upgrade. + """ + + try: + migrate_func = { + 'install': self.install, + 'upgrade': self.upgrade, + }[migrate_operation] + except KeyError: + raise ValueError(f'Invalid operation passed in {migrate_operation}') + + name = new_package_entry.name + version = new_package_entry.version + + if dockerd_sock: + # dockerd_sock is defined, so use docked_sock to connect to + # dockerd and fetch package image from it. + log.info(f'{migrate_operation} {name} from old docker library') + docker_api = DockerApi(docker.DockerClient(base_url=f'unix://{dockerd_sock}')) + + image = docker_api.get_image(old_package_entry.image_id) + + with tempfile.NamedTemporaryFile('wb') as file: + for chunk in image.save(named=True): + file.write(chunk) + + migrate_func(tarball=file.name) + else: + log.info(f'{migrate_operation} {name} version {version}') + + migrate_func(f'{name}=={version}') + + # TODO: Topological sort packages by their dependencies first. + for old_package in old_package_database: + if not old_package.installed or old_package.built_in: + continue + + log.info(f'migrating package {old_package.name}') + + new_package = self.database.get_package(old_package.name) + + if new_package.installed: + if old_package.version > new_package.version: + log.info(f'{old_package.name} package version is greater ' + f'then installed in new image: ' + f'{old_package.version} > {new_package.version}') + log.info(f'upgrading {new_package.name} to {old_package.version}') + new_package.version = old_package.version + migrate_package(old_package, new_package, 'upgrade') + else: + log.info(f'skipping {new_package.name} as installed version is newer') + elif new_package.default_reference is not None: + new_package_ref = PackageReference(new_package.name, new_package.default_reference) + package_source = self.get_package_source(package_ref=new_package_ref) + package = package_source.get_package() + new_package_default_version = package.manifest['package']['version'] + if old_package.version > new_package_default_version: + log.info(f'{old_package.name} package version is lower ' + f'then the default in new image: ' + f'{old_package.version} > {new_package_default_version}') + new_package.version = old_package.version + migrate_package(old_package, new_package, 'install') + else: + self.install(f'{new_package.name}=={new_package_default_version}') + else: + # No default version and package is not installed. + # Migrate old package same version. + new_package.version = old_package.version + migrate_package(old_package, new_package, 'install') + + self.database.commit() + + def get_installed_package(self, name: str) -> Package: + """ Get installed package by name. + + Args: + name: package name. + Returns: + Package object. + """ + + package_entry = self.database.get_package(name) + source = LocalSource(package_entry, + self.database, + self.docker, + self.metadata_resolver) + return source.get_package() + + def get_package_source(self, + package_expression: Optional[str] = None, + repository_reference: Optional[str] = None, + tarboll_path: Optional[str] = None, + package_ref: Optional[PackageReference] = None): + """ Returns PackageSource object based on input source. + + Args: + package_expression: SONiC Package expression string + repository_reference: Install from REPO[:TAG][@DIGEST] + tarboll_path: Install from image tarball + package_ref: Package reference object + Returns: + SONiC Package object. + Raises: + ValueError if no source specified. + """ + + if package_expression: + ref = parse_reference_expression(package_expression) + return self.get_package_source(package_ref=ref) + elif repository_reference: + repo_ref = DockerReference.parse(repository_reference) + repository = repo_ref['name'] + reference = repo_ref['tag'] or repo_ref['digest'] + reference = reference or 'latest' + return RegistrySource(repository, + reference, + self.database, + self.docker, + self.metadata_resolver) + elif tarboll_path: + return TarballSource(tarboll_path, + self.database, + self.docker, + self.metadata_resolver) + elif package_ref: + package_entry = self.database.get_package(package_ref.name) + + # Determine the reference if not specified. + # If package is installed assume the installed + # one is requested, otherwise look for default + # reference defined for this package. In case package + # does not have a default reference raise an error. + if package_ref.reference is None: + if package_entry.installed: + return LocalSource(package_entry, + self.database, + self.docker, + self.metadata_resolver) + if package_entry.default_reference is not None: + package_ref.reference = package_entry.default_reference + else: + raise PackageManagerError(f'No default reference tag. ' + f'Please specify the version or tag explicitly') + + return RegistrySource(package_entry.repository, + package_ref.reference, + self.database, + self.docker, + self.metadata_resolver) + else: + raise ValueError('No package source provided') + + def get_package_available_versions(self, + name: str, + all: bool = False) -> Iterable: + """ Returns a list of available versions for package. + + Args: + name: Package name. + all: If set to True will return all tags including + those which do not follow semantic versioning. + Returns: + List of versions + """ + package_info = self.database.get_package(name) + registry = self.registry_resolver.get_registry_for(package_info.repository) + available_tags = registry.tags(package_info.repository) + + def is_semantic_ver_tag(tag: str) -> bool: + try: + tag_to_version(tag) + return True + except ValueError: + pass + return False + + if all: + return available_tags + + return map(tag_to_version, filter(is_semantic_ver_tag, available_tags)) + + def is_installed(self, name: str) -> bool: + """ Returns boolean whether a package called name is installed. + + Args: + name: Package name. + Returns: + True if package is installed, False otherwise. + """ + + if not self.database.has_package(name): + return False + package_info = self.database.get_package(name) + return package_info.installed + + def get_installed_packages(self) -> Dict[str, Package]: + """ Returns a dictionary of installed packages where + keys are package names and values are package objects. + + Returns: + Installed packages dictionary. + """ + + return { + entry.name: self.get_installed_package(entry.name) + for entry in self.database if entry.installed + } + + def _migrate_package_database(self, old_package_database: PackageDatabase): + """ Performs part of package migration process. + For every package in old_package_database that is not listed in current + database add a corresponding entry to current database. """ + + for package in old_package_database: + if not self.database.has_package(package.name): + self.database.add_package(package.name, + package.repository, + package.description, + package.default_reference) + + def _get_installed_packages_and(self, package: Package) -> Dict[str, Package]: + """ Returns a dictionary of installed packages with their names as keys + adding a package provided in the argument. """ + + packages = self.get_installed_packages() + packages[package.name] = package + return packages + + def _get_installed_packages_except(self, package: Package) -> Dict[str, Package]: + """ Returns a dictionary of installed packages with their names as keys + removing a package provided in the argument. """ + + packages = self.get_installed_packages() + packages.pop(package.name) + return packages + + # TODO: Replace with "config feature" command. + # The problem with current "config feature" command + # is that it is asynchronous, thus can't be used + # for package upgrade purposes where we need to wait + # till service stops before upgrading docker image. + # It would be really handy if we could just call + # something like: "config feature state --wait" + # instead of operating on systemd service since + # this is basically a duplicated code from "hostcfgd". + def _systemctl_action(self, package: Package, action: str): + """ Execute systemctl action for a service supporting + multi-asic services. """ + + name = package.manifest['service']['name'] + host_service = package.manifest['service']['host-service'] + asic_service = package.manifest['service']['asic-service'] + single_instance = host_service or (asic_service and not self.is_multi_npu) + multi_instance = asic_service and self.is_multi_npu + + if in_chroot(): + return + + if single_instance: + run_command(f'systemctl {action} {name}') + if multi_instance: + for npu in range(self.num_npus): + run_command(f'systemctl {action} {name}@{npu}') + + @staticmethod + def _get_cli_plugin_name(package: Package): + return utils.make_python_identifier(package.name) + '.py' + + @classmethod + def _get_cli_plugin_path(cls, package: Package, command): + pkg_loader = pkgutil.get_loader(f'{command}.plugins') + if pkg_loader is None: + raise PackageManagerError(f'Failed to get plugins path for {command} CLI') + plugins_pkg_path = os.path.dirname(pkg_loader.path) + return os.path.join(plugins_pkg_path, cls._get_cli_plugin_name(package)) + + def _install_cli_plugins(self, package: Package): + for command in ('show', 'config', 'clear'): + self._install_cli_plugin(package, command) + + def _uninstall_cli_plugins(self, package: Package): + for command in ('show', 'config', 'clear'): + self._uninstall_cli_plugin(package, command) + + def _install_cli_plugin(self, package: Package, command: str): + image_plugin_path = package.manifest['cli'][command] + if not image_plugin_path: + return + host_plugin_path = self._get_cli_plugin_path(package, command) + self.docker.extract(package.entry.image_id, image_plugin_path, host_plugin_path) + + def _uninstall_cli_plugin(self, package: Package, command: str): + image_plugin_path = package.manifest['cli'][command] + if not image_plugin_path: + return + host_plugin_path = self._get_cli_plugin_path(package, command) + if os.path.exists(host_plugin_path): + os.remove(host_plugin_path) + + @staticmethod + def get_manager() -> 'PackageManager': + """ Creates and returns PackageManager instance. + + Returns: + PackageManager + """ + + docker_api = DockerApi(docker.from_env()) + registry_resolver = RegistryResolver() + return PackageManager(DockerApi(docker.from_env(), ProgressManager()), + registry_resolver, + PackageDatabase.from_file(), + MetadataResolver(docker_api, registry_resolver), + ServiceCreator(FeatureRegistry(SonicDB), SonicDB), + device_info, + filelock.FileLock(PACKAGE_MANAGER_LOCK_FILE, timeout=0)) diff --git a/sonic_package_manager/manifest.py b/sonic_package_manager/manifest.py new file mode 100644 index 0000000000..74da06a956 --- /dev/null +++ b/sonic_package_manager/manifest.py @@ -0,0 +1,210 @@ +#!/usr/bin/env python +from abc import ABC +from dataclasses import dataclass +from typing import Optional, List, Dict, Any + +from sonic_package_manager.constraint import ( + ComponentConstraints, + PackageConstraint, + VersionConstraint +) +from sonic_package_manager.errors import ManifestError +from sonic_package_manager.version import Version, VersionRange + + +class ManifestSchema: + """ ManifestSchema class describes and provides marshalling + and unmarshalling methods. + """ + + class Marshaller: + """ Base class for marshaling and un-marshaling. """ + + def marshal(self, value): + """ Validates and returns a valid manifest dictionary. + + Args: + value: input value to validate. + Returns: valid manifest node. + """ + + raise NotImplementedError + + def unmarshal(self, value): + """ Un-marshals the manifest to a dictionary. + + Args: + value: input value to validate. + Returns: valid manifest node. + """ + + raise NotImplementedError + + @dataclass + class ParsedMarshaller(Marshaller): + """ Marshaller used on types which support class method "parse" """ + + type: Any + + def marshal(self, value): + try: + return self.type.parse(value) + except ValueError as err: + raise ManifestError(f'Failed to marshal {value}: {err}') + + def unmarshal(self, value): + try: + return str(value) + except Exception as err: + raise ManifestError(f'Failed to unmarshal {value}: {err}') + + @dataclass + class DefaultMarshaller(Marshaller): + """ Default marshaller that validates if the given + value is instance of given type. """ + + type: type + + def marshal(self, value): + if not isinstance(value, self.type): + raise ManifestError(f'{value} is not of type {self.type.__name__}') + return value + + def unmarshal(self, value): + return value + + @dataclass + class ManifestNode(Marshaller, ABC): + """ + Base class for any manifest object. + + Attrs: + key: String representing the key for this object. + """ + + key: str + + @dataclass + class ManifestRoot(ManifestNode): + items: List + + def marshal(self, value: Optional[dict]): + result = {} + if value is None: + value = {} + + for item in self.items: + next_value = value.get(item.key) + result[item.key] = item.marshal(next_value) + return result + + def unmarshal(self, value): + return_value = {} + for item in self.items: + return_value[item.key] = item.unmarshal(value[item.key]) + return return_value + + @dataclass + class ManifestField(ManifestNode): + type: Any + default: Optional[Any] = None + + def marshal(self, value): + if value is None: + if self.default is not None: + return self.default + raise ManifestError(f'{self.key} is a required field but it is missing') + try: + return_value = self.type.marshal(value) + except Exception as err: + raise ManifestError(f'Failed to marshal {self.key}: {err}') + return return_value + + def unmarshal(self, value): + return self.type.unmarshal(value) + + @dataclass + class ManifestArray(ManifestNode): + type: Any + + def marshal(self, value): + if value is None: + return [] + + return_value = [] + try: + for item in value: + return_value.append(self.type.marshal(item)) + except Exception as err: + raise ManifestError(f'Failed to convert {self.key}={value} to array: {err}') + + return return_value + + def unmarshal(self, value): + return [self.type.unmarshal(item) for item in value] + + # TODO: add description for each field + SCHEMA = ManifestRoot('root', [ + ManifestField('version', ParsedMarshaller(Version), Version(1, 0, 0)), + ManifestRoot('package', [ + ManifestField('version', ParsedMarshaller(Version)), + ManifestField('name', DefaultMarshaller(str)), + ManifestField('description', DefaultMarshaller(str), ''), + ManifestField('base-os', ParsedMarshaller(ComponentConstraints), dict()), + ManifestArray('depends', ParsedMarshaller(PackageConstraint)), + ManifestArray('breaks', ParsedMarshaller(PackageConstraint)), + ManifestField('init-cfg', DefaultMarshaller(dict), dict()), + ManifestField('changelog', DefaultMarshaller(dict), dict()), + ManifestField('debug-dump', DefaultMarshaller(str), ''), + ]), + ManifestRoot('service', [ + ManifestField('name', DefaultMarshaller(str)), + ManifestArray('requires', DefaultMarshaller(str)), + ManifestArray('requisite', DefaultMarshaller(str)), + ManifestArray('wanted-by', DefaultMarshaller(str)), + ManifestArray('after', DefaultMarshaller(str)), + ManifestArray('before', DefaultMarshaller(str)), + ManifestArray('dependent', DefaultMarshaller(str)), + ManifestArray('dependent-of', DefaultMarshaller(str)), + ManifestField('post-start-action', DefaultMarshaller(str), ''), + ManifestField('pre-shutdown-action', DefaultMarshaller(str), ''), + ManifestField('asic-service', DefaultMarshaller(bool), False), + ManifestField('host-service', DefaultMarshaller(bool), True), + ManifestField('delayed', DefaultMarshaller(bool), False), + ]), + ManifestRoot('container', [ + ManifestField('privileged', DefaultMarshaller(bool), False), + ManifestArray('volumes', DefaultMarshaller(str)), + ManifestArray('mounts', ManifestRoot('mounts', [ + ManifestField('source', DefaultMarshaller(str)), + ManifestField('target', DefaultMarshaller(str)), + ManifestField('type', DefaultMarshaller(str)), + ])), + ManifestField('environment', DefaultMarshaller(dict), dict()), + ManifestArray('tmpfs', DefaultMarshaller(str)), + ]), + ManifestArray('processes', ManifestRoot('processes', [ + ManifestField('critical', DefaultMarshaller(bool)), + ManifestField('name', DefaultMarshaller(str)), + ManifestField('command', DefaultMarshaller(str)), + ])), + ManifestRoot('cli', [ + ManifestField('mandatory', DefaultMarshaller(bool), False), + ManifestField('show', DefaultMarshaller(str), ''), + ManifestField('config', DefaultMarshaller(str), ''), + ManifestField('clear', DefaultMarshaller(str), '') + ]) + ]) + + +class Manifest(dict): + """ Manifest object. """ + + SCHEMA = ManifestSchema.SCHEMA + + @classmethod + def marshal(cls, input_dict: dict): + return Manifest(cls.SCHEMA.marshal(input_dict)) + + def unmarshal(self) -> Dict: + return self.SCHEMA.unmarshal(self) diff --git a/sonic_package_manager/metadata.py b/sonic_package_manager/metadata.py new file mode 100644 index 0000000000..e5f9dbb3a5 --- /dev/null +++ b/sonic_package_manager/metadata.py @@ -0,0 +1,185 @@ +#!/usr/bin/env python + +from dataclasses import dataclass, field + +import json +import tarfile +from typing import Dict + +from sonic_package_manager.errors import MetadataError +from sonic_package_manager.manifest import Manifest +from sonic_package_manager.version import Version + + +def deep_update(dst: Dict, src: Dict) -> Dict: + """ Deep update dst dictionary with src dictionary. + + Args: + dst: Dictionary to update + src: Dictionary to update with + + Returns: + New merged dictionary. + """ + + for key, value in src.iteritems(): + if isinstance(value, dict): + node = dst.setdefault(key, {}) + deep_update(node, value) + else: + dst[key] = value + return dst + + +def translate_plain_to_tree(plain: Dict[str, str], sep='.') -> Dict: + """ Convert plain key/value dictionary into + a tree by spliting the key with '.' + + Args: + plain: Dictionary to convert into tree-like structure. + Keys in this dictionary have to be in a format: + "[key0].+", e.g: "com.azure.sonic" that + will be converted into tree like struct: + + { + "com": { + "azure": { + "sonic": {} + } + } + } + sep: Seperator string + + Returns: + Tree like structure + + """ + + res = {} + for key, value in plain.items(): + if sep not in key: + res[key] = value + continue + namespace, key = key.split(sep, 1) + res.setdefault(key, {}) + deep_update(res[key], translate_plain_to_tree({key: value})) + return res + + +@dataclass +class Metadata: + """ Package metadata object that can be retrieved from + OCI image manifest. """ + + manifest: Manifest + components: Dict[str, Version] = field(default_factory=dict) + + +class MetadataResolver: + """ Resolve metadata for package from different sources. """ + + def __init__(self, docker, registry_resolver): + self.docker = docker + self.registry_resolver = registry_resolver + + def from_local(self, image: str) -> Metadata: + """ Reads manifest from locally installed docker image. + + Args: + image: Docker image ID + Returns: + Metadata + Raises: + MetadataError + """ + + labels = self.docker.labels(image) + if labels is None: + raise MetadataError('No manifest found in image labels') + + return self.from_labels(labels) + + def from_registry(self, + repository: str, + reference: str) -> Metadata: + """ Reads manifest from remote registry. + + Args: + repository: Repository to pull image from + reference: Reference, either tag or digest + Returns: + Metadata + Raises: + MetadataError + """ + + registry = self.registry_resolver.get_registry_for(repository) + + manifest = registry.manifest(repository, reference) + digest = manifest['config']['digest'] + + blob = registry.blobs(repository, digest) + labels = blob['config']['Labels'] + if labels is None: + raise MetadataError('No manifest found in image labels') + + return self.from_labels(labels) + + def from_tarball(self, image_path: str) -> Metadata: + """ Reads manifest image tarball. + Args: + image_path: Path to image tarball. + Returns: + Manifest + Raises: + MetadataError + """ + + with tarfile.open(image_path) as image: + manifest = json.loads(image.extractfile('manifest.json').read()) + + blob = manifest[0]['Config'] + image_config = json.loads(image.extractfile(blob).read()) + labels = image_config['config']['Labels'] + if labels is None: + raise MetadataError('No manifest found in image labels') + + return self.from_labels(labels) + + @classmethod + def from_labels(cls, labels: Dict[str, str]) -> Metadata: + """ Get manifest from image labels. + + Args: + labels: key, value string pairs + Returns: + Metadata + Raises: + MetadataError + """ + + metadata_dict = translate_plain_to_tree(labels) + try: + sonic_metadata = metadata_dict['com']['azure']['sonic'] + except KeyError: + raise MetadataError('No metadata found in image labels') + + try: + manifest_string = sonic_metadata['manifest'] + except KeyError: + raise MetadataError('No manifest found in image labels') + + try: + manifest_dict = json.loads(manifest_string) + except (ValueError, TypeError) as err: + raise MetadataError(f'Failed to parse manifest JSON: {err}') + + components = {} + if 'versions' in sonic_metadata: + for component, version in sonic_metadata['versions'].items(): + try: + components[component] = Version.parse(version) + except ValueError as err: + raise MetadataError(f'Failed to parse component version: {err}') + + return Metadata(Manifest.marshal(manifest_dict), components) diff --git a/sonic_package_manager/package.py b/sonic_package_manager/package.py new file mode 100644 index 0000000000..2928f17392 --- /dev/null +++ b/sonic_package_manager/package.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python + +from dataclasses import dataclass + +from sonic_package_manager.database import PackageEntry +from sonic_package_manager.metadata import Metadata + + +@dataclass +class Package: + """ Package class is a representation of Package. + + Attributes: + entry: Package entry in package database + metadata: Metadata object for this package + manifest: Manifest for this package + components: Components versions for this package + name: Name of the package from package database + repository: Default repository to pull this package from + image_id: Docker image ID of the installed package; + It is set to None if package is not installed. + installed: Boolean flag whether package is installed or not. + build_in: Boolean flag whether package is built in or not. + + """ + + entry: PackageEntry + metadata: Metadata + + @property + def name(self): return self.entry.name + + @property + def repository(self): return self.entry.repository + + @property + def image_id(self): return self.entry.image_id + + @property + def installed(self): return self.entry.installed + + @property + def built_in(self): return self.entry.built_in + + @property + def version(self): return self.entry.version + + @property + def manifest(self): return self.metadata.manifest + + @property + def components(self): return self.metadata.components + diff --git a/sonic_package_manager/progress.py b/sonic_package_manager/progress.py new file mode 100644 index 0000000000..5258ebab98 --- /dev/null +++ b/sonic_package_manager/progress.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python + +import enlighten + +BAR_FMT = '{desc}{desc_pad}{percentage:3.0f}%|{bar}| {count:{len_total}.2f}/{total:.2f}{unit_pad}{unit} ' + \ + '[{elapsed}<{eta}, {rate:.2f}{unit_pad}{unit}/s]' + +COUNTER_FMT = '{desc}{desc_pad}{count:.1f} {unit}{unit_pad}' + \ + '[{elapsed}, {rate:.2f}{unit_pad}{unit}/s]{fill}' + + +class ProgressManager: + """ ProgressManager is used for creating multiple progress bars + which nicely interact with logging and prints. """ + + def __init__(self): + self.manager = enlighten.get_manager() + self.pbars = {} + + def __enter__(self): + return self.manager.__enter__() + + def __exit__(self, exc_type, exc_val, exc_tb): + return self.manager.__exit__(exc_type, exc_val, exc_tb) + + def new(self, id: str, *args, **kwargs): + """ Creates new progress bar with id. + Args: + id: progress bar identifier + *args: pass arguments for progress bar creation + **kwargs: pass keyword arguments for progress bar creation. + """ + + if 'bar_format' not in kwargs: + kwargs['bar_format'] = BAR_FMT + if 'counter_format' not in kwargs: + kwargs['counter_format'] = COUNTER_FMT + + self.pbars[id] = self.manager.counter(*args, **kwargs) + + def get(self, id: str): + """ Returns progress bar by id. + Args: + id: progress bar identifier + Returns: + Progress bar. + """ + + return self.pbars[id] + + def __contains__(self, id): + return id in self.pbars diff --git a/sonic_package_manager/reference.py b/sonic_package_manager/reference.py new file mode 100644 index 0000000000..9c4d8e825c --- /dev/null +++ b/sonic_package_manager/reference.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python + +import re +from dataclasses import dataclass +from typing import Optional + + +@dataclass +class PackageReference: + """ PackageReference is a package version constraint. """ + + name: str + reference: Optional[str] = None + + def __str__(self): + return f'{self.name} {self.reference}' + + @staticmethod + def parse(expression: str) -> 'PackageReference': + REQUIREMENT_SPECIFIER_RE = \ + r'(?P[A-Za-z0-9_-]+)(?P@(?P.*))' + + match = re.match(REQUIREMENT_SPECIFIER_RE, expression) + if match is None: + raise ValueError(f'Invalid reference specifier {expression}') + groupdict = match.groupdict() + name = groupdict.get('name') + reference = groupdict.get('reference') + + return PackageReference(name, reference) diff --git a/sonic_package_manager/registry.py b/sonic_package_manager/registry.py new file mode 100644 index 0000000000..bf4308efa0 --- /dev/null +++ b/sonic_package_manager/registry.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python +import json +from dataclasses import dataclass +from typing import List, Dict + +import requests +import www_authenticate +from docker_image import reference +from prettyprinter import pformat + +from sonic_package_manager.logger import log +from sonic_package_manager.utils import DockerReference + + +class AuthenticationServiceError(Exception): + """ Exception class for errors related to authentication. """ + + pass + + +class AuthenticationService: + """ AuthenticationService provides an authentication tokens. """ + + @staticmethod + def get_token(realm, service, scope) -> str: + """ Retrieve an authentication token. + + Args: + realm: Realm: url to request token. + service: service to request token for. + scope: scope to requests token for. + Returns: + token value as a string. + """ + + log.debug(f'getting authentication token: realm={realm} service={service} scope={scope}') + + response = requests.get(f'{realm}?scope={scope}&service={service}') + if response.status_code != requests.codes.ok: + raise AuthenticationServiceError(f'Failed to retrieve token') + + content = json.loads(response.content) + token = content['token'] + expires_in = content['expires_in'] + + log.debug(f'authentication token for realm={realm} service={service} scope={scope}: ' + f'token={token} expires_in={expires_in}') + + return token + + +@dataclass +class RegistryApiError(Exception): + """ Class for registry related errors. """ + + msg: str + response: requests.Response + + def __str__(self): + code = self.response.status_code + content = self.response.content.decode() + try: + content = json.loads(content) + except ValueError: + pass + return f'{self.msg}: code: {code} details: {pformat(content)}' + + +class Registry: + """ Provides a Docker registry interface. """ + + MIME_DOCKER_MANIFEST = 'application/vnd.docker.distribution.manifest.v2+json' + + def __init__(self, host: str): + self.url = host + + @staticmethod + def _execute_get_request(url, headers): + response = requests.get(url, headers=headers) + if response.status_code == requests.codes.unauthorized: + # Get authentication details from headers + # Registry should tell how to authenticate + www_authenticate_details = response.headers['Www-Authenticate'] + log.debug(f'unauthorized: retrieving authentication details ' + f'from response headers {www_authenticate_details}') + bearer = www_authenticate.parse(www_authenticate_details)['bearer'] + token = AuthenticationService.get_token(**bearer) + headers['Authorization'] = f'Bearer {token}' + # Repeat request + response = requests.get(url, headers=headers) + return response + + def _get_base_url(self, repository: str): + return f'{self.url}/v2/{repository}' + + def tags(self, repository: str) -> List[str]: + log.debug(f'getting tags for {repository}') + + _, repository = reference.Reference.split_docker_domain(repository) + headers = {'Accept': 'application/json'} + url = f'{self._get_base_url(repository)}/tags/list' + response = self._execute_get_request(url, headers) + if response.status_code != requests.codes.ok: + raise RegistryApiError(f'Failed to retrieve tags from {repository}', response) + + content = json.loads(response.content) + log.debug(f'tags list api response: f{content}') + + return content['tags'] + + def manifest(self, repository: str, ref: str) -> Dict: + log.debug(f'getting manifest for {repository}:{ref}') + + _, repository = reference.Reference.split_docker_domain(repository) + headers = {'Accept': self.MIME_DOCKER_MANIFEST} + url = f'{self._get_base_url(repository)}/manifests/{ref}' + response = self._execute_get_request(url, headers) + + if response.status_code != requests.codes.ok: + raise RegistryApiError(f'Failed to retrieve manifest for {repository}:{ref}', response) + + content = json.loads(response.content) + log.debug(f'manifest content for {repository}:{ref}: {content}') + + return content + + def blobs(self, repository: str, digest: str): + log.debug(f'retrieving blob for {repository}:{digest}') + + _, repository = reference.Reference.split_docker_domain(repository) + headers = {'Accept': self.MIME_DOCKER_MANIFEST} + url = f'{self._get_base_url(repository)}/blobs/{digest}' + response = self._execute_get_request(url, headers) + if response.status_code != requests.codes.ok: + raise RegistryApiError(f'Failed to retrieve blobs for {repository}:{digest}', response) + content = json.loads(response.content) + + log.debug(f'retrieved blob for {repository}:{digest}: {content}') + return content + + +class RegistryResolver: + """ Returns a registry object based on the input repository reference + string. """ + + DockerHubRegistry = Registry('https://index.docker.io') + + def __init__(self): + pass + + def get_registry_for(self, ref: str) -> Registry: + domain, _ = DockerReference.split_docker_domain(ref) + if domain == reference.DEFAULT_DOMAIN: + return self.DockerHubRegistry + # TODO: support insecure registries + return Registry(f'https://{domain}') diff --git a/sonic_package_manager/service_creator/__init__.py b/sonic_package_manager/service_creator/__init__.py new file mode 100644 index 0000000000..e2af81ceb5 --- /dev/null +++ b/sonic_package_manager/service_creator/__init__.py @@ -0,0 +1,3 @@ +#!/usr/bin/env python + +ETC_SONIC_PATH = '/etc/sonic' diff --git a/sonic_package_manager/service_creator/creator.py b/sonic_package_manager/service_creator/creator.py new file mode 100644 index 0000000000..f62d0a3074 --- /dev/null +++ b/sonic_package_manager/service_creator/creator.py @@ -0,0 +1,339 @@ +#!/usr/bin/env python +import contextlib +import os +import stat +import subprocess +from typing import Dict + +import jinja2 as jinja2 +from prettyprinter import pformat + +from sonic_package_manager.logger import log +from sonic_package_manager.package import Package +from sonic_package_manager.service_creator import ETC_SONIC_PATH +from sonic_package_manager.service_creator.feature import FeatureRegistry +from sonic_package_manager.service_creator.utils import in_chroot + +SERVICE_FILE_TEMPLATE = 'sonic.service.j2' +TIMER_UNIT_TEMPLATE = 'timer.unit.j2' + +SYSTEMD_LOCATION = '/usr/lib/systemd/system' + +SERVICE_MGMT_SCRIPT_TEMPLATE = 'service_mgmt.sh.j2' +SERVICE_MGMT_SCRIPT_LOCATION = '/usr/local/bin' + +DOCKER_CTL_SCRIPT_TEMPLATE = 'docker_image_ctl.j2' +DOCKER_CTL_SCRIPT_LOCATION = '/usr/bin' + +MONIT_CONF_TEMPLATE = 'monit.conf.j2' +MONIT_CONF_LOCATION = '/etc/monit/conf.d/' + +DEBUG_DUMP_SCRIPT_TEMPLATE = 'dump.sh.j2' +DEBUG_DUMP_SCRIPT_LOCATION = '/usr/local/bin/debug-dump/' + +TEMPLATES_PATH = '/usr/share/sonic/templates' + + +class ServiceCreatorError(Exception): + pass + + +def render_template(in_template: str, + outfile: str, + render_ctx: Dict, + executable: bool = False): + """ Template renderer helper routine. + Args: + in_template: Input file with template content + outfile: Output file to render template to + render_ctx: Dictionary used to generate jinja2 template + executable: Set executable bit on rendered file + """ + + log.debug(f'Rendering {in_template} to {outfile} with {pformat(render_ctx)}') + + with open(in_template, 'r') as instream: + template = jinja2.Template(instream.read()) + + with open(outfile, 'w') as outstream: + outstream.write(template.render(**render_ctx)) + + if executable: + set_executable_bit(outfile) + + +def get_tmpl_path(template_name: str) -> str: + """ Returns a path to a template. + Args: + template_name: Template file name. + """ + + return os.path.join(TEMPLATES_PATH, template_name) + + +def set_executable_bit(filepath): + """ Sets +x on filepath. """ + + st = os.stat(filepath) + os.chmod(filepath, st.st_mode | stat.S_IEXEC) + + +def run_command(command: str): + """ Run arbitrary bash command. + Args: + command: String command to execute as bash script + Raises: + PackageManagerError: Raised when the command return code + is not 0. + """ + + log.debug(f'running command: {command}') + + proc = subprocess.Popen(command, + shell=True, + executable='/bin/bash', + stdout=subprocess.PIPE) + (out, _) = proc.communicate() + if proc.returncode != 0: + raise ServiceCreatorError(f'Failed to execute "{command}"') + + +class ServiceCreator: + """ Creates and registers services in SONiC based on the package + manifest. """ + + def __init__(self, feature_registry: FeatureRegistry, sonic_db): + self.feature_registry = feature_registry + self.sonic_db = sonic_db + + def create(self, + package: Package, + register_feature=True, + state='enabled', + owner='local'): + try: + self.generate_container_mgmt(package) + self.generate_service_mgmt(package) + self.update_dependent_list_file(package) + self.generate_systemd_service(package) + self.generate_monit_conf(package) + self.generate_dump_script(package) + + self.set_initial_config(package) + + self.post_operation_hook() + + if register_feature: + self.feature_registry.register(package.manifest, + state, owner) + except (Exception, KeyboardInterrupt): + self.remove(package, not register_feature) + raise + + def remove(self, package: Package, deregister_feature=True): + name = package.manifest['service']['name'] + + def remove_file(path): + if os.path.exists(path): + os.remove(path) + log.info(f'removed {path}') + + remove_file(os.path.join(MONIT_CONF_LOCATION, f'monit_{name}')) + remove_file(os.path.join(SYSTEMD_LOCATION, f'{name}.service')) + remove_file(os.path.join(SYSTEMD_LOCATION, f'{name}@.service')) + remove_file(os.path.join(SERVICE_MGMT_SCRIPT_LOCATION, f'{name}.sh')) + remove_file(os.path.join(DOCKER_CTL_SCRIPT_LOCATION, f'{name}.sh')) + remove_file(os.path.join(DEBUG_DUMP_SCRIPT_LOCATION, f'{name}')) + + self.update_dependent_list_file(package, remove=True) + + self.post_operation_hook() + + if deregister_feature: + self.feature_registry.deregister(package.manifest['service']['name']) + + def post_operation_hook(self): + if not in_chroot(): + run_command('systemctl daemon-reload') + run_command('systemctl reload monit') + + def generate_container_mgmt(self, package: Package): + image_id = package.image_id + name = package.manifest['service']['name'] + container_spec = package.manifest['container'] + script_path = os.path.join(DOCKER_CTL_SCRIPT_LOCATION, f'{name}.sh') + script_template = get_tmpl_path(DOCKER_CTL_SCRIPT_TEMPLATE) + run_opt = [] + + if container_spec['privileged']: + run_opt.append('--privileged') + + run_opt.append('-t') + + for volume in container_spec['volumes']: + run_opt.append(f'-v {volume}') + + for mount in container_spec['mounts']: + mount_type, source, target = mount['type'], mount['source'], mount['target'] + run_opt.append(f'--mount type={mount_type},source={source},target={target}') + + for tmpfs_mount in container_spec['tmpfs']: + run_opt.append(f'--tmpfs {tmpfs_mount}') + + for env_name, value in container_spec['environment'].items(): + run_opt.append(f'-e {env_name}={value}') + + run_opt = ' '.join(run_opt) + render_ctx = { + 'docker_container_name': name, + 'docker_image_id': image_id, + 'docker_image_run_opt': run_opt, + } + render_template(script_template, script_path, render_ctx, executable=True) + log.info(f'generated {script_path}') + + def generate_service_mgmt(self, package: Package): + name = package.manifest['service']['name'] + multi_instance_services = self.feature_registry.get_multi_instance_features() + script_path = os.path.join(SERVICE_MGMT_SCRIPT_LOCATION, f'{name}.sh') + scrip_template = get_tmpl_path(SERVICE_MGMT_SCRIPT_TEMPLATE) + render_ctx = { + 'source': get_tmpl_path(SERVICE_MGMT_SCRIPT_TEMPLATE), + 'manifest': package.manifest.unmarshal(), + 'multi_instance_services': multi_instance_services, + } + render_template(scrip_template, script_path, render_ctx, executable=True) + log.info(f'generated {script_path}') + + def generate_systemd_service(self, package: Package): + name = package.manifest['service']['name'] + multi_instance_services = self.feature_registry.get_multi_instance_features() + + template = get_tmpl_path(SERVICE_FILE_TEMPLATE) + template_vars = { + 'source': get_tmpl_path(SERVICE_FILE_TEMPLATE), + 'manifest': package.manifest.unmarshal(), + 'multi_instance': False, + 'multi_instance_services': multi_instance_services, + } + output_file = os.path.join(SYSTEMD_LOCATION, f'{name}.service') + render_template(template, output_file, template_vars) + log.info(f'generated {output_file}') + + if package.manifest['service']['asic-service']: + output_file = os.path.join(SYSTEMD_LOCATION, f'{name}@.service') + template_vars['multi_instance'] = True + render_template(template, output_file, template_vars) + log.info(f'generated {output_file}') + + if package.manifest['service']['delayed']: + template_vars = { + 'source': get_tmpl_path(TIMER_UNIT_TEMPLATE), + 'manifest': package.manifest.unmarshal(), + 'multi_instance': False, + } + output_file = os.path.join(SYSTEMD_LOCATION, f'{name}.timer') + template = os.path.join(TEMPLATES_PATH, TIMER_UNIT_TEMPLATE) + render_template(template, output_file, template_vars) + log.info(f'generated {output_file}') + + if package.manifest['service']['asic-service']: + output_file = os.path.join(SYSTEMD_LOCATION, f'{name}@.timer') + template_vars['multi_instance'] = True + render_template(template, output_file, template_vars) + log.info(f'generated {output_file}') + + def generate_monit_conf(self, package: Package): + name = package.manifest['service']['name'] + processes = package.manifest['processes'] + output_filename = os.path.join(MONIT_CONF_LOCATION, f'monit_{name}') + render_template(get_tmpl_path(MONIT_CONF_TEMPLATE), output_filename, + {'source': get_tmpl_path(MONIT_CONF_TEMPLATE), + 'feature': name, + 'processes': processes}) + log.info(f'generated {output_filename}') + + def update_dependent_list_file(self, package: Package, remove=False): + name = package.manifest['service']['name'] + dependent_of = package.manifest['service']['dependent-of'] + host_service = package.manifest['service']['host-service'] + asic_service = package.manifest['service']['asic-service'] + + def update_dependent(service, name, multi_inst): + if multi_inst: + filename = f'{service}_multi_inst_dependent' + else: + filename = f'{service}_dependent' + + filepath = os.path.join(ETC_SONIC_PATH, filename) + + dependent_services = set() + if os.path.exists(filepath): + with open(filepath) as fp: + dependent_services.update({line.strip() for line in fp.readlines()}) + if remove: + with contextlib.suppress(KeyError): + dependent_services.remove(name) + else: + dependent_services.add(name) + with open(filepath, 'w') as fp: + fp.write('\n'.join(dependent_services)) + + for service in dependent_of: + if host_service: + update_dependent(service, name, multi_inst=False) + if asic_service: + update_dependent(service, name, multi_inst=True) + + def generate_dump_script(self, package): + name = package.manifest['service']['name'] + + if not package.manifest['package']['debug-dump']: + return + + if not os.path.exists(DEBUG_DUMP_SCRIPT_LOCATION): + os.mkdir(DEBUG_DUMP_SCRIPT_LOCATION) + + scrip_template = os.path.join(TEMPLATES_PATH, DEBUG_DUMP_SCRIPT_TEMPLATE) + script_path = os.path.join(DEBUG_DUMP_SCRIPT_LOCATION, f'{name}') + render_ctx = { + 'source': get_tmpl_path(SERVICE_MGMT_SCRIPT_TEMPLATE), + 'manifest': package.manifest.unmarshal(), + } + render_template(scrip_template, script_path, render_ctx, executable=True) + log.info(f'generated {script_path}') + + def set_initial_config(self, package): + init_cfg = package.manifest['package']['init-cfg'] + + def get_tables(table_name): + tables = [] + + running_table = self.sonic_db.running_table(table_name) + if running_table is not None: + tables.append(running_table) + + persistent_table = self.sonic_db.persistent_table(table_name) + if persistent_table is not None: + tables.append(persistent_table) + + initial_table = self.sonic_db.initial_table(table_name) + if initial_table is not None: + tables.append(initial_table) + + return tables + + for tablename, content in init_cfg.items(): + if not isinstance(content, dict): + continue + + tables = get_tables(tablename) + + for key in content: + for table in tables: + cfg = content[key] + exists, old_fvs = table.get(key) + if exists: + cfg.update(old_fvs) + fvs = list(cfg.items()) + table.set(key, fvs) diff --git a/sonic_package_manager/service_creator/feature.py b/sonic_package_manager/service_creator/feature.py new file mode 100644 index 0000000000..4df06384d2 --- /dev/null +++ b/sonic_package_manager/service_creator/feature.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python + +""" This module implements new feature registration/de-registration in SONiC system. """ + +from typing import Dict, Type + +from sonic_package_manager.manifest import Manifest +from sonic_package_manager.service_creator.sonic_db import SonicDB + +FEATURE = 'FEATURE' +DEFAULT_FEATURE_CONFIG = { + 'state': 'disabled', + 'auto_restart': 'enabled', + 'high_mem_alert': 'disabled', + 'set_owner': 'local' +} + + +class FeatureRegistry: + """ FeatureRegistry class provides an interface to + register/de-register new feature persistently. """ + + def __init__(self, sonic_db: Type[SonicDB]): + self._sonic_db = sonic_db + + def register(self, + manifest: Manifest, + state: str = 'disabled', + owner: str = 'local'): + name = manifest['service']['name'] + for table in self._get_tables(): + cfg_entries = self.get_default_feature_entries(state, owner) + non_cfg_entries = self.get_non_configurable_feature_entries(manifest) + + exists, current_cfg = table.get(name) + + new_cfg = cfg_entries.copy() + # Override configurable entries with CONFIG DB data. + new_cfg = {**new_cfg, **dict(current_cfg)} + # Override CONFIG DB data with non configurable entries. + new_cfg = {**new_cfg, **non_cfg_entries} + + table.set(name, list(new_cfg.items())) + + def deregister(self, name: str): + for table in self._get_tables(): + table._del(name) + + def is_feature_enabled(self, name: str) -> bool: + """ Returns whether the feature is current enabled + or not. Accesses running CONFIG DB. If no running CONFIG_DB + table is found in tables returns False. """ + + running_db_table = self._sonic_db.running_table(FEATURE) + if running_db_table is None: + return False + + exists, cfg = running_db_table.get(name) + if not exists: + return False + cfg = dict(cfg) + return cfg.get('state').lower() == 'enabled' + + def get_multi_instance_features(self): + res = [] + init_db_table = self._sonic_db.initial_table(FEATURE) + for feature in init_db_table.keys(): + exists, cfg = init_db_table.get(feature) + assert exists + cfg = dict(cfg) + asic_flag = str(cfg.get('has_per_asic_scope', 'False')) + if asic_flag.lower() == 'true': + res.append(feature) + return res + + @staticmethod + def get_default_feature_entries(state=None, owner=None) -> Dict[str, str]: + """ Get configurable feature table entries: + e.g. 'state', 'auto_restart', etc. """ + + cfg = DEFAULT_FEATURE_CONFIG.copy() + if state: + cfg['state'] = state + if owner: + cfg['set_owner'] = owner + return cfg + + @staticmethod + def get_non_configurable_feature_entries(manifest) -> Dict[str, str]: + """ Get non-configurable feature table entries: e.g. 'has_timer' """ + + return { + 'has_per_asic_scope': str(manifest['service']['asic-service']), + 'has_global_scope': str(manifest['service']['host-service']), + 'has_timer': str(manifest['service']['delayed']), + } + + def _get_tables(self): + tables = [] + running = self._sonic_db.running_table(FEATURE) + if running is not None: # it's Ok if there is no database container running + tables.append(running) + persistent = self._sonic_db.persistent_table(FEATURE) + if persistent is not None: # it's Ok if there is no config_db.json + tables.append(persistent) + tables.append(self._sonic_db.initial_table(FEATURE)) # init_cfg.json is must + + return tables diff --git a/sonic_package_manager/service_creator/sonic_db.py b/sonic_package_manager/service_creator/sonic_db.py new file mode 100644 index 0000000000..a9ba837ab1 --- /dev/null +++ b/sonic_package_manager/service_creator/sonic_db.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python +import contextlib +import json +import os + +from swsscommon import swsscommon + +from sonic_package_manager.service_creator import ETC_SONIC_PATH +from sonic_package_manager.service_creator.utils import in_chroot + +CONFIG_DB = 'CONFIG_DB' +CONFIG_DB_JSON = os.path.join(ETC_SONIC_PATH, 'config_db.json') +INIT_CFG_JSON = os.path.join(ETC_SONIC_PATH, 'init_cfg.json') + + +class FileDbTable: + """ swsscommon.Table adapter for persistent DBs. """ + + def __init__(self, file, table): + self._file = file + self._table = table + + def keys(self): + with open(self._file) as stream: + config = json.load(stream) + return config.get(self._table, {}).keys() + + def get(self, key): + with open(self._file) as stream: + config = json.load(stream) + + table = config.get(self._table, {}) + exists = key in table + fvs_dict = table.get(key, {}) + fvs = list(fvs_dict.items()) + return exists, fvs + + def set(self, key, fvs): + with open(self._file) as stream: + config = json.load(stream) + + table = config.setdefault(self._table, {}) + table.update({key: dict(fvs)}) + + with open(self._file, 'w') as stream: + json.dump(config, stream, indent=4) + + def _del(self, key): + with open(self._file) as stream: + config = json.load(stream) + + with contextlib.suppress(KeyError): + config[self._table].pop(key) + + with open(self._file, 'w') as stream: + json.dump(config, stream, indent=4) + + +class SonicDB: + """ Store different DB access objects for + running DB and also for persistent and initial + configs. """ + + _running = None + + @classmethod + def running_table(cls, table): + """ Returns running DB table. """ + + # In chroot we can connect to a running + # DB via TCP socket, we should ignore this case. + if in_chroot(): + return None + + if cls._running is None: + try: + cls._running = swsscommon.DBConnector(CONFIG_DB, 0) + except RuntimeError: + # Failed to connect to DB. + return None + + return swsscommon.Table(cls._running, table) + + @classmethod + def persistent_table(cls, table): + """ Returns persistent DB table. """ + + if not os.path.exists(CONFIG_DB_JSON): + return None + + return FileDbTable(CONFIG_DB_JSON, table) + + @classmethod + def initial_table(cls, table): + """ Returns initial DB table. """ + + return FileDbTable(INIT_CFG_JSON, table) diff --git a/sonic_package_manager/service_creator/utils.py b/sonic_package_manager/service_creator/utils.py new file mode 100644 index 0000000000..cdeeb17abb --- /dev/null +++ b/sonic_package_manager/service_creator/utils.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python + +import os + + +def in_chroot() -> bool: + """ Verify if we are running in chroot or not + by comparing root / mount point device id and inode + with init process - /proc/1/root mount point device + id and inode. If those match we are not chroot-ed + otherwise we are. """ + + root_stat = os.stat('/') + init_root_stat = os.stat('/proc/1/root') + + return (root_stat.st_dev, root_stat.st_ino) != \ + (init_root_stat.st_dev, init_root_stat.st_ino) diff --git a/sonic_package_manager/source.py b/sonic_package_manager/source.py new file mode 100644 index 0000000000..4720b5be9e --- /dev/null +++ b/sonic_package_manager/source.py @@ -0,0 +1,178 @@ +#!/usr/bin/env python3 + +from sonic_package_manager.database import PackageDatabase, PackageEntry +from sonic_package_manager.dockerapi import DockerApi, get_repository_from_image +from sonic_package_manager.metadata import Metadata, MetadataResolver +from sonic_package_manager.package import Package + + +class PackageSource(object): + """ PackageSource abstracts the way manifest is read + and image is retrieved based on different image sources. + (i.e from registry, from tarball or locally installed) """ + + def __init__(self, + database: PackageDatabase, + docker: DockerApi, + metadata_resolver: MetadataResolver): + self.database = database + self.docker = docker + self.metadata_resolver = metadata_resolver + + def get_metadata(self) -> Metadata: + """ Returns package manifest. + Child class has to implement this method. + + Returns: + Metadata + """ + raise NotImplementedError + + def install_image(self): + """ Install image based on package source. + Child class has to implement this method. + + Returns: + Docker Image object. + """ + + raise NotImplementedError + + def install(self, package: Package): + """ Install image based on package source, + record installation infromation in PackageEntry.. + + Args: + package: SONiC Package + """ + + image = self.install_image() + package.entry.image_id = image.id + # if no repository is defined for this package + # get repository from image + if not package.repository: + package.entry.repository = get_repository_from_image(image) + + def uninstall(self, package: Package): + """ Uninstall image. + + Args: + package: SONiC Package + """ + + self.docker.rmi(package.image_id) + package.entry.image_id = None + + def get_package(self) -> Package: + """ Returns SONiC Package based on manifest. + + Returns: + SONiC Package + """ + + metadata = self.get_metadata() + manifest = metadata.manifest + + name = manifest['package']['name'] + description = manifest['package']['description'] + + repository = None + + if self.database.has_package(name): + # inherit package database info + package = self.database.get_package(name) + repository = package.repository + description = description or package.description + + return Package( + PackageEntry( + name, + repository, + description, + ), + metadata + ) + + +class TarballSource(PackageSource): + """ TarballSource implements PackageSource + for locally existing image saved as tarball. """ + + def __init__(self, + tarball_path: str, + database: PackageDatabase, + docker: DockerApi, + metadata_resolver: MetadataResolver): + super().__init__(database, + docker, + metadata_resolver) + self.tarball_path = tarball_path + + def get_metadata(self) -> Metadata: + """ Returns manifest read from tarball. """ + + return self.metadata_resolver.from_tarball(self.tarball_path) + + def install_image(self): + """ Installs image from local tarball source. """ + + return self.docker.load(self.tarball_path) + + +class RegistrySource(PackageSource): + """ RegistrySource implements PackageSource + for packages that are pulled from registry. """ + + def __init__(self, + repository: str, + reference: str, + database: PackageDatabase, + docker: DockerApi, + metadata_resolver: MetadataResolver): + super().__init__(database, + docker, + metadata_resolver) + self.repository = repository + self.reference = reference + + def get_metadata(self) -> Metadata: + """ Returns manifest read from registry. """ + + return self.metadata_resolver.from_registry(self.repository, + self.reference) + + def install_image(self): + """ Installs image from registry. """ + + return self.docker.pull(self.repository, self.reference) + + +class LocalSource(PackageSource): + """ LocalSource accesses local docker library to retrieve manifest + but does not implement installation of the image. """ + + def __init__(self, + entry: PackageEntry, + database: PackageDatabase, + docker: DockerApi, + metadata_resolver: MetadataResolver): + super().__init__(database, + docker, + metadata_resolver) + self.entry = entry + + def get_metadata(self) -> Metadata: + """ Returns manifest read from locally installed Docker. """ + + image = self.entry.image_id + + if self.entry.built_in: + # Built-in (installed not via sonic-package-manager) + # won't have image_id in database. Using their + # repository name as image. + image = f'{self.entry.repository}:latest' + + return self.metadata_resolver.from_local(image) + + def get_package(self) -> Package: + return Package(self.entry, self.get_metadata()) diff --git a/sonic_package_manager/utils.py b/sonic_package_manager/utils.py new file mode 100644 index 0000000000..410947dd24 --- /dev/null +++ b/sonic_package_manager/utils.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python + +import keyword +import re + +from docker_image.reference import Reference + +DockerReference = Reference + + +def make_python_identifier(string): + """ + Takes an arbitrary string and creates a valid Python identifier. + + Identifiers must follow the convention outlined here: + https://docs.python.org/2/reference/lexical_analysis.html#identifiers + """ + + # create a working copy (and make it lowercase, while we're at it) + s = string.lower() + + # remove leading and trailing whitespace + s = s.strip() + + # Make spaces into underscores + s = re.sub('[\\s\\t\\n]+', '_', s) + + # Remove invalid characters + s = re.sub('[^0-9a-zA-Z_]', '', s) + + # Remove leading characters until we find a letter or underscore + s = re.sub('^[^a-zA-Z_]+', '', s) + + # Check that the string is not a python identifier + while s in keyword.kwlist: + if re.match(".*?_\d+$", s): + i = re.match(".*?_(\d+)$", s).groups()[0] + s = s.strip('_'+i) + '_'+str(int(i)+1) + else: + s += '_1' + + return s diff --git a/sonic_package_manager/version.py b/sonic_package_manager/version.py new file mode 100644 index 0000000000..e5a5623d3b --- /dev/null +++ b/sonic_package_manager/version.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python + +""" Version and helpers routines. """ + +import semver + +Version = semver.Version +VersionRange = semver.VersionRange + + +def version_to_tag(ver: Version) -> str: + """ Converts the version to Docker compliant tag string. """ + + return str(ver).replace('+', '_') + + +def tag_to_version(tag: str) -> Version: + """ Converts the version to Docker compliant tag string. """ + + try: + return Version.parse(tag.replace('_', '+')) + except ValueError as err: + raise ValueError(f'Failed to convert {tag} to version string: {err}') diff --git a/tests/sonic_package_manager/conftest.py b/tests/sonic_package_manager/conftest.py new file mode 100644 index 0000000000..bf787bdab4 --- /dev/null +++ b/tests/sonic_package_manager/conftest.py @@ -0,0 +1,379 @@ +#!/usr/bin/env python + +from dataclasses import dataclass +from unittest import mock +from unittest.mock import Mock, MagicMock + +import pytest +from docker_image.reference import Reference + +from sonic_package_manager.database import PackageDatabase, PackageEntry +from sonic_package_manager.manager import DockerApi, PackageManager +from sonic_package_manager.manifest import Manifest +from sonic_package_manager.metadata import Metadata, MetadataResolver +from sonic_package_manager.registry import RegistryResolver +from sonic_package_manager.version import Version +from sonic_package_manager.service_creator.creator import * + + +@pytest.fixture +def mock_docker_api(): + docker = MagicMock(DockerApi) + + @dataclass + class Image: + id: str + + @property + def attrs(self): + return {'RepoTags': [self.id]} + + def pull(repo, ref): + return Image(f'{repo}:latest') + + def load(filename): + return Image(filename) + + docker.pull = MagicMock(side_effect=pull) + docker.load = MagicMock(side_effect=load) + + yield docker + + +@pytest.fixture +def mock_registry_resolver(): + yield Mock(RegistryResolver) + + +@pytest.fixture +def mock_metadata_resolver(): + yield Mock(MetadataResolver) + + +@pytest.fixture +def mock_feature_registry(): + yield MagicMock() + + +@pytest.fixture +def mock_service_creator(): + yield Mock() + + +@pytest.fixture +def mock_sonic_db(): + yield Mock() + + +@pytest.fixture +def fake_metadata_resolver(): + class FakeMetadataResolver: + def __init__(self): + self.metadata_store = {} + self.add('docker-database', 'latest', 'database', '1.0.0') + self.add('docker-orchagent', 'latest', 'swss', '1.0.0', + components={ + 'libswsscommon': Version.parse('1.0.0'), + 'libsairedis': Version.parse('1.0.0') + } + ) + self.add('Azure/docker-test', '1.6.0', 'test-package', '1.6.0') + self.add('Azure/docker-test-2', '1.5.0', 'test-package-2', '1.5.0') + self.add('Azure/docker-test-2', '2.0.0', 'test-package-2', '2.0.0') + self.add('Azure/docker-test-3', 'latest', 'test-package-3', '1.6.0') + self.add('Azure/docker-test-3', '1.5.0', 'test-package-3', '1.5.0') + self.add('Azure/docker-test-3', '1.6.0', 'test-package-3', '1.6.0') + self.add('Azure/docker-test-4', '1.5.0', 'test-package-4', '1.5.0') + self.add('Azure/docker-test-5', '1.5.0', 'test-package-5', '1.5.0') + self.add('Azure/docker-test-5', '1.9.0', 'test-package-5', '1.9.0') + self.add('Azure/docker-test-6', '1.5.0', 'test-package-6', '1.5.0') + self.add('Azure/docker-test-6', '1.9.0', 'test-package-6', '1.9.0') + self.add('Azure/docker-test-6', '2.0.0', 'test-package-6', '2.0.0') + self.add('Azure/docker-test-6', 'latest', 'test-package-6', '1.5.0') + + def from_registry(self, repository: str, reference: str): + manifest = Manifest.marshal(self.metadata_store[repository][reference]['manifest']) + components = self.metadata_store[repository][reference]['components'] + return Metadata(manifest, components) + + def from_local(self, image: str): + ref = Reference.parse(image) + manifest = Manifest.marshal(self.metadata_store[ref['name']][ref['tag']]['manifest']) + components = self.metadata_store[ref['name']][ref['tag']]['components'] + return Metadata(manifest, components) + + def from_tarball(self, filepath: str) -> Manifest: + path, ref = filepath.split(':') + manifest = Manifest.marshal(self.metadata_store[path][ref]['manifest']) + components = self.metadata_store[path][ref]['components'] + return Metadata(manifest, components) + + def add(self, repo, reference, name, version, components=None): + repo_dict = self.metadata_store.setdefault(repo, {}) + repo_dict[reference] = { + 'manifest': { + 'package': { + 'version': version, + 'name': name, + 'base-os': {}, + }, + 'service': { + 'name': name, + } + }, + 'components': components or {}, + } + + yield FakeMetadataResolver() + + +@pytest.fixture +def fake_device_info(): + class FakeDeviceInfo: + def __init__(self): + self.multi_npu = True + self.num_npus = 1 + self.version_info = { + 'libswsscommon': '1.0.0', + } + + def is_multi_npu(self): + return self.multi_npu + + def get_num_npus(self): + return self.num_npus + + def get_sonic_version_info(self): + return self.version_info + + yield FakeDeviceInfo() + + +def add_package(content, metadata_resolver, repository, reference, **kwargs): + metadata = metadata_resolver.from_registry(repository, reference) + name = metadata.manifest['package']['name'] + version = metadata.manifest['package']['version'] + installed = kwargs.get('installed', False) + built_in = kwargs.get('built-in', False) + + if installed and not built_in and 'image_id' not in kwargs: + kwargs['image_id'] = f'{repository}:{reference}' + + if installed and 'version' not in kwargs: + kwargs['version'] = version + + content[name] = PackageEntry(name, repository, **kwargs) + + +@pytest.fixture +def fake_db(fake_metadata_resolver): + content = {} + + add_package( + content, + fake_metadata_resolver, + 'docker-database', + 'latest', + description='SONiC database service', + default_reference='1.0.0', + installed=True, + built_in=True + ) + add_package( + content, + fake_metadata_resolver, + 'docker-orchagent', + 'latest', + description='SONiC switch state service', + default_reference='1.0.0', + installed=True, + built_in=True + ) + add_package( + content, + fake_metadata_resolver, + 'Azure/docker-test', + '1.6.0', + description='SONiC Package Manager Test Package', + default_reference='1.6.0', + installed=False, + built_in=False + ) + add_package( + content, + fake_metadata_resolver, + 'Azure/docker-test-2', + '1.5.0', + description='SONiC Package Manager Test Package #2', + default_reference='1.5.0', + installed=False, + built_in=False + ) + add_package( + content, + fake_metadata_resolver, + 'Azure/docker-test-3', + '1.5.0', + description='SONiC Package Manager Test Package #3', + default_reference='1.5.0', + installed=True, + built_in=False + ) + add_package( + content, + fake_metadata_resolver, + 'Azure/docker-test-5', + '1.9.0', + description='SONiC Package Manager Test Package #5', + default_reference='1.9.0', + installed=False, + built_in=False + ) + add_package( + content, + fake_metadata_resolver, + 'Azure/docker-test-6', + '1.5.0', + description='SONiC Package Manager Test Package #6', + default_reference='1.5.0', + installed=False, + built_in=False + ) + + yield PackageDatabase(content) + + +@pytest.fixture +def fake_db_for_migration(fake_metadata_resolver): + content = {} + add_package( + content, + fake_metadata_resolver, + 'docker-database', + 'latest', + description='SONiC database service', + default_reference='1.0.0', + installed=True, + built_in=True + ) + add_package( + content, + fake_metadata_resolver, + 'docker-orchagent', + 'latest', + description='SONiC switch state service', + default_reference='1.0.0', + installed=True, + built_in=True + ) + add_package( + content, + fake_metadata_resolver, + 'Azure/docker-test', + '1.6.0', + description='SONiC Package Manager Test Package', + default_reference='1.6.0', + installed=False, + built_in=False + ) + add_package( + content, + fake_metadata_resolver, + 'Azure/docker-test-2', + '2.0.0', + description='SONiC Package Manager Test Package #2', + default_reference='2.0.0', + installed=False, + built_in=False + ) + add_package( + content, + fake_metadata_resolver, + 'Azure/docker-test-3', + '1.6.0', + description='SONiC Package Manager Test Package #3', + default_reference='1.6.0', + installed=True, + built_in=False + ) + add_package( + content, + fake_metadata_resolver, + 'Azure/docker-test-4', + '1.5.0', + description='SONiC Package Manager Test Package #4', + default_reference='1.5.0', + installed=True, + built_in=False + ) + add_package( + content, + fake_metadata_resolver, + 'Azure/docker-test-5', + '1.5.0', + description='SONiC Package Manager Test Package #5', + default_reference='1.5.0', + installed=True, + built_in=False + ) + add_package( + content, + fake_metadata_resolver, + 'Azure/docker-test-6', + '2.0.0', + description='SONiC Package Manager Test Package #6', + default_reference='2.0.0', + installed=True, + built_in=False + ) + + yield PackageDatabase(content) + + +@pytest.fixture() +def sonic_fs(fs): + fs.create_file('/proc/1/root') + fs.create_dir(ETC_SONIC_PATH) + fs.create_dir(SYSTEMD_LOCATION) + fs.create_dir(DOCKER_CTL_SCRIPT_LOCATION) + fs.create_dir(SERVICE_MGMT_SCRIPT_LOCATION) + fs.create_dir(MONIT_CONF_LOCATION) + fs.create_file(os.path.join(TEMPLATES_PATH, SERVICE_FILE_TEMPLATE)) + fs.create_file(os.path.join(TEMPLATES_PATH, TIMER_UNIT_TEMPLATE)) + fs.create_file(os.path.join(TEMPLATES_PATH, SERVICE_MGMT_SCRIPT_TEMPLATE)) + fs.create_file(os.path.join(TEMPLATES_PATH, DOCKER_CTL_SCRIPT_TEMPLATE)) + fs.create_file(os.path.join(TEMPLATES_PATH, MONIT_CONF_TEMPLATE)) + fs.create_file(os.path.join(TEMPLATES_PATH, DEBUG_DUMP_SCRIPT_TEMPLATE)) + yield fs + + +@pytest.fixture(autouse=True) +def patch_pkgutil(): + with mock.patch('pkgutil.get_loader'): + yield + + +@pytest.fixture +def package_manager(mock_docker_api, + mock_registry_resolver, + mock_service_creator, + fake_metadata_resolver, + fake_db, + fake_device_info): + yield PackageManager(mock_docker_api, mock_registry_resolver, + fake_db, fake_metadata_resolver, + mock_service_creator, + fake_device_info, + MagicMock()) + + +@pytest.fixture +def anything(): + """ Fixture that returns Any object that can be used in + assert_called_*_with to match any object passed. """ + + class Any: + def __eq__(self, other): + return True + + yield Any() diff --git a/tests/sonic_package_manager/test_cli.py b/tests/sonic_package_manager/test_cli.py new file mode 100644 index 0000000000..695d8cba58 --- /dev/null +++ b/tests/sonic_package_manager/test_cli.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python + +from click.testing import CliRunner + +from sonic_package_manager import main + + +def test_show_changelog(package_manager, fake_metadata_resolver): + """ Test case for "sonic-package-manager package show changelog [NAME]" """ + + runner = CliRunner() + changelog = { + "1.0.0": { + "changes": ["Initial release"], + "author": "Stepan Blyshchak", + "email": "stepanb@nvidia.com", + "date": "Mon, 25 May 2020 12:24:30 +0300" + }, + "1.1.0": { + "changes": [ + "Added functionality", + "Bug fixes" + ], + "author": "Stepan Blyshchak", + "email": "stepanb@nvidia.com", + "date": "Fri, 23 Oct 2020 12:26:08 +0300" + } + } + manifest = fake_metadata_resolver.metadata_store['Azure/docker-test']['1.6.0']['manifest'] + manifest['package']['changelog'] = changelog + + expected_output = """\ +1.0.0: + + • Initial release + + Stepan Blyshchak (stepanb@nvidia.com) Mon, 25 May 2020 12:24:30 +0300 + +1.1.0: + + • Added functionality + • Bug fixes + + Stepan Blyshchak (stepanb@nvidia.com) Fri, 23 Oct 2020 12:26:08 +0300 + +""" + + result = runner.invoke(main.show.commands['package'].commands['changelog'], + ['test-package'], obj=package_manager) + + assert result.exit_code == 0 + assert result.output == expected_output + + +def test_show_changelog_no_changelog(package_manager): + """ Test case for "sonic-package-manager package show changelog [NAME]" + when there is no changelog provided by package. """ + + runner = CliRunner() + result = runner.invoke(main.show.commands['package'].commands['changelog'], ['test-package'], obj=package_manager) + + assert result.exit_code == 1 + assert result.output == 'Failed to print package changelog: No changelog for package test-package\n' diff --git a/tests/sonic_package_manager/test_constraint.py b/tests/sonic_package_manager/test_constraint.py new file mode 100644 index 0000000000..2e7067ef63 --- /dev/null +++ b/tests/sonic_package_manager/test_constraint.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python +from sonic_package_manager import version +from sonic_package_manager.constraint import PackageConstraint +from sonic_package_manager.version import Version, VersionRange + + +def test_constraint(): + package_constraint = PackageConstraint.parse('swss>1.0.0') + assert package_constraint.name == 'swss' + assert not package_constraint.constraint.allows(Version.parse('0.9.1')) + assert package_constraint.constraint.allows(Version.parse('1.1.1')) + + +def test_constraint_range(): + package_constraint = PackageConstraint.parse('swss^1.2.0') + assert package_constraint.name == 'swss' + assert not package_constraint.constraint.allows(Version.parse('1.1.1')) + assert package_constraint.constraint.allows(Version.parse('1.2.5')) + assert not package_constraint.constraint.allows(Version.parse('2.0.1')) + + +def test_constraint_strict(): + package_constraint = PackageConstraint.parse('swss==1.2.0') + assert package_constraint.name == 'swss' + assert not package_constraint.constraint.allows(Version.parse('1.1.1')) + assert package_constraint.constraint.allows(Version.parse('1.2.0')) + + +def test_constraint_match(): + package_constraint = PackageConstraint.parse('swss==1.2*.*') + assert package_constraint.name == 'swss' + assert not package_constraint.constraint.allows(Version.parse('1.1.1')) + assert package_constraint.constraint.allows(Version.parse('1.2.0')) + + +def test_constraint_multiple(): + package_constraint = PackageConstraint.parse('swss>1.2.0,<3.0.0,!=2.2.2') + assert package_constraint.name == 'swss' + assert not package_constraint.constraint.allows(Version.parse('2.2.2')) + assert not package_constraint.constraint.allows(Version.parse('3.2.0')) + assert not package_constraint.constraint.allows(Version.parse('0.2.0')) + assert package_constraint.constraint.allows(Version.parse('2.2.3')) + assert package_constraint.constraint.allows(Version.parse('1.2.3')) + + +def test_constraint_only_name(): + package_constraint = PackageConstraint.parse('swss') + assert package_constraint.name == 'swss' + assert package_constraint.constraint == VersionRange() + + +def test_constraint_from_dict(): + package_constraint = PackageConstraint.parse({ + 'name': 'swss', + 'version': '^1.0.0', + 'components': { + 'libswsscommon': '^1.1.0', + }, + }) + assert package_constraint.name == 'swss' + assert package_constraint.constraint.allows(Version.parse('1.0.0')) + assert not package_constraint.constraint.allows(Version.parse('2.0.0')) + assert package_constraint.components['libswsscommon'].allows(Version.parse('1.2.0')) + assert not package_constraint.components['libswsscommon'].allows(Version.parse('1.0.0')) + assert not package_constraint.components['libswsscommon'].allows(Version.parse('2.0.0')) + + +def test_version_to_tag(): + assert version.version_to_tag(Version.parse('1.0.0-rc0')) == '1.0.0-rc0' + assert version.version_to_tag(Version.parse('1.0.0-rc0+152')) == '1.0.0-rc0_152' + + +def test_tag_to_version(): + assert str(version.tag_to_version('1.0.0-rc0_152')) == '1.0.0-rc0+152' + assert str(version.tag_to_version('1.0.0-rc0')) == '1.0.0-rc0' diff --git a/tests/sonic_package_manager/test_database.py b/tests/sonic_package_manager/test_database.py new file mode 100644 index 0000000000..1c565d6f4c --- /dev/null +++ b/tests/sonic_package_manager/test_database.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python + +import pytest + +from sonic_package_manager.database import PackageEntry +from sonic_package_manager.errors import ( + PackageNotFoundError, + PackageAlreadyExistsError, + PackageManagerError +) +from sonic_package_manager.version import Version + + +def test_database_get_package(fake_db): + swss_package = fake_db.get_package('swss') + assert swss_package.installed + assert swss_package.built_in + assert swss_package.repository == 'docker-orchagent' + assert swss_package.default_reference == '1.0.0' + assert swss_package.version == Version(1, 0, 0) + + +def test_database_get_package_not_builtin(fake_db): + test_package = fake_db.get_package('test-package') + assert not test_package.installed + assert not test_package.built_in + assert test_package.repository == 'Azure/docker-test' + assert test_package.default_reference == '1.6.0' + assert test_package.version is None + + +def test_database_get_package_not_existing(fake_db): + with pytest.raises(PackageNotFoundError): + fake_db.get_package('abc') + + +def test_database_add_package(fake_db): + fake_db.add_package('test-package-99', 'Azure/docker-test-99') + test_package = fake_db.get_package('test-package-99') + assert not test_package.installed + assert not test_package.built_in + assert test_package.repository == 'Azure/docker-test-99' + assert test_package.default_reference is None + assert test_package.version is None + + +def test_database_add_package_existing(fake_db): + with pytest.raises(PackageAlreadyExistsError): + fake_db.add_package('swss', 'Azure/docker-orchagent') + + +def test_database_update_package(fake_db): + test_package = fake_db.get_package('test-package-2') + test_package.installed = True + test_package.version = Version(1, 2, 3) + fake_db.update_package(test_package) + test_package = fake_db.get_package('test-package-2') + assert test_package.installed + assert test_package.version == Version(1, 2, 3) + + +def test_database_update_package_non_existing(fake_db): + test_package = PackageEntry('abc', 'abc') + with pytest.raises(PackageNotFoundError): + fake_db.update_package(test_package) + + +def test_database_remove_package(fake_db): + fake_db.remove_package('test-package') + assert not fake_db.has_package('test-package') + + +def test_database_remove_package_non_existing(fake_db): + with pytest.raises(PackageNotFoundError): + fake_db.remove_package('non-existing-package') + + +def test_database_remove_package_installed(fake_db): + with pytest.raises(PackageManagerError, + match='Package test-package-3 is installed, ' + 'uninstall it first'): + fake_db.remove_package('test-package-3') + + +def test_database_remove_package_built_in(fake_db): + with pytest.raises(PackageManagerError, + match='Package swss is built-in, ' + 'cannot remove it'): + fake_db.remove_package('swss') diff --git a/tests/sonic_package_manager/test_manager.py b/tests/sonic_package_manager/test_manager.py new file mode 100644 index 0000000000..256832118e --- /dev/null +++ b/tests/sonic_package_manager/test_manager.py @@ -0,0 +1,321 @@ +#!/usr/bin/env python +from unittest.mock import Mock, call + +import pytest + +from sonic_package_manager.errors import * +from sonic_package_manager.version import Version + + +def test_installation_not_installed(package_manager): + package_manager.install('test-package') + + +def test_installation_already_installed(package_manager): + with pytest.raises(PackageInstallationError, + match='swss is already installed'): + package_manager.install('swss') + + +def test_installation_dependencies(package_manager, fake_metadata_resolver, mock_docker_api): + manifest = fake_metadata_resolver.metadata_store['Azure/docker-test']['1.6.0']['manifest'] + manifest['package']['depends'] = ['swss^2.0.0'] + with pytest.raises(PackageInstallationError, + match='Package test-package requires swss>=2.0.0,<3.0.0 ' + 'but version 1.0.0 is installed'): + package_manager.install('test-package') + + +def test_installation_dependencies_missing_package(package_manager, fake_metadata_resolver): + manifest = fake_metadata_resolver.metadata_store['Azure/docker-test']['1.6.0']['manifest'] + manifest['package']['depends'] = ['missing-package>=1.0.0'] + with pytest.raises(PackageInstallationError, + match='Package test-package requires ' + 'missing-package>=1.0.0 but it is not installed'): + package_manager.install('test-package') + + +def test_installation_dependencies_satisfied(package_manager, fake_metadata_resolver): + manifest = fake_metadata_resolver.metadata_store['Azure/docker-test']['1.6.0']['manifest'] + manifest['package']['depends'] = ['database>=1.0.0', 'swss>=1.0.0'] + package_manager.install('test-package') + + +def test_installation_components_dependencies_satisfied(package_manager, fake_metadata_resolver): + metadata = fake_metadata_resolver.metadata_store['Azure/docker-test']['1.6.0'] + manifest = metadata['manifest'] + metadata['components'] = { + 'libswsscommon': Version.parse('1.1.0') + } + manifest['package']['depends'] = [ + { + 'name': 'swss', + 'version': '>=1.0.0', + 'components': { + 'libswsscommon': '^1.0.0', + }, + }, + ] + package_manager.install('test-package') + + +def test_installation_components_dependencies_not_satisfied(package_manager, fake_metadata_resolver): + metadata = fake_metadata_resolver.metadata_store['Azure/docker-test']['1.6.0'] + manifest = metadata['manifest'] + metadata['components'] = { + 'libswsscommon': Version.parse('1.1.0') + } + manifest['package']['depends'] = [ + { + 'name': 'swss', + 'version': '>=1.0.0', + 'components': { + 'libswsscommon': '^1.1.0', + }, + }, + ] + with pytest.raises(PackageInstallationError, + match='Package test-package requires libswsscommon >=1.1.0,<2.0.0 ' + 'in package swss>=1.0.0 but version 1.0.0 is installed'): + package_manager.install('test-package') + + +def test_installation_components_dependencies_implicit(package_manager, fake_metadata_resolver): + metadata = fake_metadata_resolver.metadata_store['Azure/docker-test']['1.6.0'] + manifest = metadata['manifest'] + metadata['components'] = { + 'libswsscommon': Version.parse('2.1.0') + } + manifest['package']['depends'] = [ + { + 'name': 'swss', + 'version': '>=1.0.0', + }, + ] + with pytest.raises(PackageInstallationError, + match='Package test-package requires libswsscommon >=2.1.0,<3.0.0 ' + 'in package swss>=1.0.0 but version 1.0.0 is installed'): + package_manager.install('test-package') + + +def test_installation_components_dependencies_explicitely_allowed(package_manager, fake_metadata_resolver): + metadata = fake_metadata_resolver.metadata_store['Azure/docker-test']['1.6.0'] + manifest = metadata['manifest'] + metadata['components'] = { + 'libswsscommon': Version.parse('2.1.0') + } + manifest['package']['depends'] = [ + { + 'name': 'swss', + 'version': '>=1.0.0', + 'components': { + 'libswsscommon': '>=1.0.0,<3.0.0' + } + }, + ] + package_manager.install('test-package') + + +def test_installation_breaks(package_manager, fake_metadata_resolver): + manifest = fake_metadata_resolver.metadata_store['Azure/docker-test']['1.6.0']['manifest'] + manifest['package']['breaks'] = ['swss^1.0.0'] + with pytest.raises(PackageInstallationError, + match='Package test-package conflicts with ' + 'swss>=1.0.0,<2.0.0 but version 1.0.0 is installed'): + package_manager.install('test-package') + + +def test_installation_breaks_missing_package(package_manager, fake_metadata_resolver): + manifest = fake_metadata_resolver.metadata_store['Azure/docker-test']['1.6.0']['manifest'] + manifest['package']['breaks'] = ['missing-package^1.0.0'] + package_manager.install('test-package') + + +def test_installation_breaks_not_installed_package(package_manager, fake_metadata_resolver): + manifest = fake_metadata_resolver.metadata_store['Azure/docker-test']['1.6.0']['manifest'] + manifest['package']['breaks'] = ['test-package-2^1.0.0'] + package_manager.install('test-package') + + +def test_installation_base_os_constraint(package_manager, fake_metadata_resolver): + manifest = fake_metadata_resolver.metadata_store['Azure/docker-test']['1.6.0']['manifest'] + manifest['package']['base-os']['libswsscommon'] = '>=2.0.0' + with pytest.raises(PackageSonicRequirementError, + match='Package test-package requires base OS component libswsscommon ' + 'version >=2.0.0 while the installed version is 1.0.0'): + package_manager.install('test-package') + + +def test_installation_base_os_constraint_satisfied(package_manager, fake_metadata_resolver): + manifest = fake_metadata_resolver.metadata_store['Azure/docker-test']['1.6.0']['manifest'] + manifest['package']['base-os']['libswsscommon'] = '>=1.0.0' + package_manager.install('test-package') + + +def test_installation_cli_plugin(package_manager, fake_metadata_resolver, anything): + manifest = fake_metadata_resolver.metadata_store['Azure/docker-test']['1.6.0']['manifest'] + manifest['cli']= {'show': '/cli/plugin.py'} + package_manager._install_cli_plugins = Mock() + package_manager.install('test-package') + package_manager._install_cli_plugins.assert_called_once_with(anything) + + +def test_installation_cli_plugin_skipped(package_manager, fake_metadata_resolver, anything): + manifest = fake_metadata_resolver.metadata_store['Azure/docker-test']['1.6.0']['manifest'] + manifest['cli']= {'show': '/cli/plugin.py'} + package_manager._install_cli_plugins = Mock() + package_manager.install('test-package', skip_cli_plugin_installation=True) + package_manager._install_cli_plugins.assert_not_called() + + +def test_installation_cli_plugin_is_mandatory_but_skipped(package_manager, fake_metadata_resolver): + manifest = fake_metadata_resolver.metadata_store['Azure/docker-test']['1.6.0']['manifest'] + manifest['cli']= {'mandatory': True} + with pytest.raises(PackageManagerError, + match='CLI is mandatory for package test-package but ' + 'it was requested to be not installed'): + package_manager.install('test-package', skip_cli_plugin_installation=True) + + +def test_installation(package_manager, mock_docker_api, anything): + package_manager.install('test-package') + mock_docker_api.pull.assert_called_once_with('Azure/docker-test', '1.6.0') + + +def test_installation_using_reference(package_manager, + fake_metadata_resolver, + mock_docker_api, + anything): + ref = 'sha256:9780f6d83e45878749497a6297ed9906c19ee0cc48cc88dc63827564bb8768fd' + metadata = fake_metadata_resolver.metadata_store['Azure/docker-test']['1.6.0'] + fake_metadata_resolver.metadata_store['Azure/docker-test'][ref] = metadata + + package_manager.install(f'test-package@{ref}') + mock_docker_api.pull.assert_called_once_with('Azure/docker-test', f'{ref}') + + +def test_manager_installation_tag(package_manager, + mock_docker_api, + anything): + package_manager.install(f'test-package==1.6.0') + mock_docker_api.pull.assert_called_once_with('Azure/docker-test', '1.6.0') + + +def test_installation_from_file(package_manager, mock_docker_api, sonic_fs): + sonic_fs.create_file('Azure/docker-test:1.6.0') + package_manager.install(tarball='Azure/docker-test:1.6.0') + mock_docker_api.load.assert_called_once_with('Azure/docker-test:1.6.0') + + +def test_installation_from_registry(package_manager, mock_docker_api): + package_manager.install(repotag='Azure/docker-test:1.6.0') + mock_docker_api.pull.assert_called_once_with('Azure/docker-test', '1.6.0') + + +def test_installation_from_registry_using_digest(package_manager, mock_docker_api, fake_metadata_resolver): + ref = 'sha256:9780f6d83e45878749497a6297ed9906c19ee0cc48cc88dc63827564bb8768fd' + metadata = fake_metadata_resolver.metadata_store['Azure/docker-test']['1.6.0'] + fake_metadata_resolver.metadata_store['Azure/docker-test'][ref] = metadata + + ref = 'sha256:9780f6d83e45878749497a6297ed9906c19ee0cc48cc88dc63827564bb8768fd' + package_manager.install(repotag=f'Azure/docker-test@{ref}') + mock_docker_api.pull.assert_called_once_with('Azure/docker-test', ref) + + +def test_installation_from_file_known_package(package_manager, fake_db, sonic_fs): + repository = fake_db.get_package('test-package').repository + sonic_fs.create_file('Azure/docker-test:1.6.0') + package_manager.install(tarball='Azure/docker-test:1.6.0') + # locally installed package does not override already known package repository + assert repository == fake_db.get_package('test-package').repository + + +def test_installation_from_file_unknown_package(package_manager, fake_db, sonic_fs): + assert not fake_db.has_package('test-package-4') + sonic_fs.create_file('Azure/docker-test-4:1.5.0') + package_manager.install(tarball='Azure/docker-test-4:1.5.0') + assert fake_db.has_package('test-package-4') + + +def test_upgrade_from_file_known_package(package_manager, fake_db, sonic_fs): + repository = fake_db.get_package('test-package-6').repository + # install older version from repository + package_manager.install('test-package-6==1.5.0') + # upgrade from file + sonic_fs.create_file('Azure/docker-test-6:2.0.0') + package_manager.upgrade(tarball='Azure/docker-test-6:2.0.0') + # locally installed package does not override already known package repository + assert repository == fake_db.get_package('test-package-6').repository + + +def test_installation_non_default_owner(package_manager, anything, mock_service_creator): + package_manager.install('test-package', default_owner='kube') + mock_service_creator.create.assert_called_once_with(anything, state='disabled', owner='kube') + + +def test_installation_enabled(package_manager, anything, mock_service_creator): + package_manager.install('test-package', enable=True) + mock_service_creator.create.assert_called_once_with(anything, state='enabled', owner='local') + + +def test_installation_fault(package_manager, mock_docker_api, mock_service_creator): + # make 'tag' to fail + mock_service_creator.create = Mock(side_effect=Exception('Failed to create service')) + # 'rmi' is called on rollback + mock_docker_api.rmi = Mock(side_effect=Exception('Failed to remove image')) + # assert that the rollback does not hide the original failure. + with pytest.raises(Exception, match='Failed to create service'): + package_manager.install('test-package') + mock_docker_api.rmi.assert_called_once() + + +def test_installation_package_with_description(package_manager, fake_metadata_resolver): + package_entry = package_manager.database.get_package('test-package') + description = package_entry.description + references = fake_metadata_resolver.metadata_store[package_entry.repository] + manifest = references[package_entry.default_reference]['manifest'] + new_description = description + ' changed description ' + manifest['package']['description'] = new_description + package_manager.install('test-package') + package_entry = package_manager.database.get_package('test-package') + description = package_entry.description + assert description == new_description + + +def test_manager_installation_version_range(package_manager): + with pytest.raises(PackageManagerError, + match='Can only install specific version. ' + 'Use only following expression "test-package==" ' + 'to install specific version'): + package_manager.install(f'test-package>=1.6.0') + + +def test_manager_upgrade(package_manager, sonic_fs): + package_manager.install('test-package-6==1.5.0') + package_manager.upgrade('test-package-6==2.0.0') + + upgraded_package = package_manager.get_installed_package('test-package-6') + assert upgraded_package.entry.version == Version(2, 0, 0) + + +def test_manager_migration(package_manager, fake_db_for_migration): + package_manager.install = Mock() + package_manager.upgrade = Mock() + package_manager.migrate_packages(fake_db_for_migration) + + # test-package-3 was installed but there is a newer version installed + # in fake_db_for_migration, asserting for upgrade + package_manager.upgrade.assert_has_calls([call('test-package-3==1.6.0')], any_order=True) + + package_manager.install.assert_has_calls([ + # test-package-4 was not present in DB at all, but it is present and installed in + # fake_db_for_migration, thus asserting that it is going to be installed. + call('test-package-4==1.5.0'), + # test-package-5 1.5.0 was installed in fake_db_for_migration but the default + # in current db is 1.9.0, assert that migration will install the newer version. + call('test-package-5==1.9.0'), + # test-package-6 2.0.0 was installed in fake_db_for_migration but the default + # in current db is 1.5.0, assert that migration will install the newer version. + call('test-package-6==2.0.0')], + any_order=True + ) diff --git a/tests/sonic_package_manager/test_manifest.py b/tests/sonic_package_manager/test_manifest.py new file mode 100644 index 0000000000..f6028e0437 --- /dev/null +++ b/tests/sonic_package_manager/test_manifest.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python + +import pytest + +from sonic_package_manager.manifest import Manifest, ManifestError +from sonic_package_manager.version import VersionRange + + +def test_manifest_v1_defaults(): + manifest = Manifest.marshal({'package': {'name': 'test', + 'version': '1.0.0'}, + 'service': {'name': 'test'}}) + assert manifest['package']['depends'] == [] + assert manifest['package']['breaks'] == [] + assert manifest['package']['base-os'] == dict() + assert not manifest['service']['asic-service'] + assert manifest['service']['host-service'] + + +def test_manifest_v1_invalid_version(): + with pytest.raises(ManifestError): + Manifest.marshal({'package': {'version': 'abc', 'name': 'test'}, + 'service': {'name': 'test'}}) + + +def test_manifest_v1_invalid_package_constraint(): + with pytest.raises(ManifestError): + Manifest.marshal({'package': {'name': 'test', 'version': '1.0.0', + 'depends': ['swss>a']}, + 'service': {'name': 'test'}}) + + +def test_manifest_v1_service_spec(): + manifest = Manifest.marshal({'package': {'name': 'test', + 'version': '1.0.0'}, + 'service': {'name': 'test', 'asic-service': True}}) + assert manifest['service']['asic-service'] + + +def test_manifest_v1_mounts(): + manifest = Manifest.marshal({'version': '1.0.0', 'package': {'name': 'test', + 'version': '1.0.0'}, + 'service': {'name': 'cpu-report'}, + 'container': {'privileged': True, + 'mounts': [{'source': 'a', 'target': 'b', 'type': 'bind'}]}}) + assert manifest['container']['mounts'][0]['source'] == 'a' + assert manifest['container']['mounts'][0]['target'] == 'b' + assert manifest['container']['mounts'][0]['type'] == 'bind' + + +def test_manifest_v1_mounts_invalid(): + with pytest.raises(ManifestError): + Manifest.marshal({'version': '1.0.0', 'package': {'name': 'test', 'version': '1.0.0'}, + 'service': {'name': 'cpu-report'}, + 'container': {'privileged': True, + 'mounts': [{'not-source': 'a', 'target': 'b', 'type': 'bind'}]}}) + + +def test_manifest_v1_unmarshal(): + manifest_json_input = {'package': {'name': 'test', 'version': '1.0.0', + 'depends': ['swss>1.0.0']}, + 'service': {'name': 'test'}} + manifest = Manifest.marshal(manifest_json_input) + manifest_json = manifest.unmarshal() + for key, section in manifest_json_input.items(): + for field, value in section.items(): + assert manifest_json[key][field] == value diff --git a/tests/sonic_package_manager/test_metadata.py b/tests/sonic_package_manager/test_metadata.py new file mode 100644 index 0000000000..4636c18282 --- /dev/null +++ b/tests/sonic_package_manager/test_metadata.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python +import contextlib +from unittest.mock import Mock, MagicMock + +from sonic_package_manager.database import PackageEntry +from sonic_package_manager.errors import MetadataError +from sonic_package_manager.metadata import MetadataResolver +from sonic_package_manager.version import Version + + +def test_metadata_resolver_local(mock_registry_resolver, mock_docker_api): + metadata_resolver = MetadataResolver(mock_docker_api, mock_registry_resolver) + # it raises exception because mock manifest is not a valid manifest + # but this is not a test objective, so just suppress the error. + with contextlib.suppress(MetadataError): + metadata_resolver.from_local('image') + mock_docker_api.labels.assert_called_once() + + +def test_metadata_resolver_remote(mock_registry_resolver, mock_docker_api): + metadata_resolver = MetadataResolver(mock_docker_api, mock_registry_resolver) + mock_registry = MagicMock() + mock_registry.manifest = MagicMock(return_value={'config': {'digest': 'some-digest'}}) + + def return_mock_registry(repository): + return mock_registry + + mock_registry_resolver.get_registry_for = Mock(side_effect=return_mock_registry) + # it raises exception because mock manifest is not a valid manifest + # but this is not a test objective, so just suppress the error. + with contextlib.suppress(MetadataError): + metadata_resolver.from_registry('test-repository', '1.2.0') + mock_registry_resolver.get_registry_for.assert_called_once_with('test-repository') + mock_registry.manifest.assert_called_once_with('test-repository', '1.2.0') + mock_registry.blobs.assert_called_once_with('test-repository', 'some-digest') + mock_docker_api.labels.assert_not_called() diff --git a/tests/sonic_package_manager/test_reference.py b/tests/sonic_package_manager/test_reference.py new file mode 100644 index 0000000000..c986632c43 --- /dev/null +++ b/tests/sonic_package_manager/test_reference.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python +import pytest + +from sonic_package_manager.reference import PackageReference + + +def test_reference(): + package_constraint = PackageReference.parse( + 'swss@sha256:9780f6d83e45878749497a6297ed9906c19ee0cc48cc88dc63827564bb8768fd' + ) + assert package_constraint.name == 'swss' + assert package_constraint.reference == 'sha256:9780f6d83e45878749497a6297ed9906c19ee0cc48cc88dc63827564bb8768fd' + + +def test_reference_invalid(): + with pytest.raises(ValueError): + PackageReference.parse('swssfdsf') diff --git a/tests/sonic_package_manager/test_registry.py b/tests/sonic_package_manager/test_registry.py new file mode 100644 index 0000000000..0d82499df3 --- /dev/null +++ b/tests/sonic_package_manager/test_registry.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python + +from sonic_package_manager.registry import RegistryResolver + + +def test_get_registry_for(): + resolver = RegistryResolver() + registry = resolver.get_registry_for('debian') + assert registry is resolver.DockerHubRegistry + registry = resolver.get_registry_for('Azure/sonic') + assert registry is resolver.DockerHubRegistry + registry = resolver.get_registry_for('registry-server:5000/docker') + assert registry.url == 'https://registry-server:5000' + registry = resolver.get_registry_for('registry-server.com/docker') + assert registry.url == 'https://registry-server.com' diff --git a/tests/sonic_package_manager/test_service_creator.py b/tests/sonic_package_manager/test_service_creator.py new file mode 100644 index 0000000000..e69b4447f8 --- /dev/null +++ b/tests/sonic_package_manager/test_service_creator.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python +import os +from unittest.mock import Mock, MagicMock + +import pytest + +from sonic_package_manager.database import PackageEntry +from sonic_package_manager.manifest import Manifest +from sonic_package_manager.metadata import Metadata +from sonic_package_manager.package import Package +from sonic_package_manager.service_creator.creator import ( + ServiceCreator, + ETC_SONIC_PATH, DOCKER_CTL_SCRIPT_LOCATION, + SERVICE_MGMT_SCRIPT_LOCATION, SYSTEMD_LOCATION, MONIT_CONF_LOCATION, DEBUG_DUMP_SCRIPT_LOCATION +) +from sonic_package_manager.service_creator.feature import FeatureRegistry + + +@pytest.fixture +def manifest(): + return Manifest.marshal({ + 'package': { + 'name': 'test', + 'version': '1.0.0', + }, + 'service': { + 'name': 'test', + 'requires': ['database'], + 'after': ['database', 'swss', 'syncd'], + 'before': ['ntp-config'], + 'dependent-of': ['swss'], + 'asic-service': False, + 'host-service': True, + }, + 'container': { + 'privileged': True, + 'volumes': [ + '/etc/sonic:/etc/sonic:ro' + ] + } + }) + + +def test_service_creator(sonic_fs, manifest, mock_feature_registry, mock_sonic_db): + creator = ServiceCreator(mock_feature_registry, mock_sonic_db) + entry = PackageEntry('test', 'azure/sonic-test') + package = Package(entry, Metadata(manifest)) + creator.create(package) + + assert sonic_fs.exists(os.path.join(ETC_SONIC_PATH, 'swss_dependent')) + assert sonic_fs.exists(os.path.join(DOCKER_CTL_SCRIPT_LOCATION, 'test.sh')) + assert sonic_fs.exists(os.path.join(SERVICE_MGMT_SCRIPT_LOCATION, 'test.sh')) + assert sonic_fs.exists(os.path.join(SYSTEMD_LOCATION, 'test.service')) + assert sonic_fs.exists(os.path.join(MONIT_CONF_LOCATION, 'monit_test')) + + +def test_service_creator_with_timer_unit(sonic_fs, manifest, mock_feature_registry, mock_sonic_db): + creator = ServiceCreator(mock_feature_registry, mock_sonic_db) + entry = PackageEntry('test', 'azure/sonic-test') + package = Package(entry, Metadata(manifest)) + creator.create(package) + + assert not sonic_fs.exists(os.path.join(SYSTEMD_LOCATION, 'test.timer')) + + manifest['service']['delayed'] = True + package = Package(entry, Metadata(manifest)) + creator.create(package) + + assert sonic_fs.exists(os.path.join(SYSTEMD_LOCATION, 'test.timer')) + + +def test_service_creator_with_debug_dump(sonic_fs, manifest, mock_feature_registry, mock_sonic_db): + creator = ServiceCreator(mock_feature_registry, mock_sonic_db) + entry = PackageEntry('test', 'azure/sonic-test') + package = Package(entry, Metadata(manifest)) + creator.create(package) + + assert not sonic_fs.exists(os.path.join(DEBUG_DUMP_SCRIPT_LOCATION, 'test')) + + manifest['package']['debug-dump'] = '/some/command' + package = Package(entry, Metadata(manifest)) + creator.create(package) + + assert sonic_fs.exists(os.path.join(DEBUG_DUMP_SCRIPT_LOCATION, 'test')) + + +def test_service_creator_initial_config(sonic_fs, manifest, mock_feature_registry, mock_sonic_db): + mock_table = Mock() + mock_table.get = Mock(return_value=(True, (('field_2', 'original_value_2'),))) + mock_sonic_db.initial_table = Mock(return_value=mock_table) + mock_sonic_db.persistent_table = Mock(return_value=mock_table) + mock_sonic_db.running_table = Mock(return_value=mock_table) + + creator = ServiceCreator(mock_feature_registry, mock_sonic_db) + + entry = PackageEntry('test', 'azure/sonic-test') + package = Package(entry, Metadata(manifest)) + creator.create(package) + + assert not sonic_fs.exists(os.path.join(DEBUG_DUMP_SCRIPT_LOCATION, 'test')) + + manifest['package']['init-cfg'] = { + 'TABLE_A': { + 'key_a': { + 'field_1': 'value_1', + 'field_2': 'value_2' + }, + }, + } + package = Package(entry, Metadata(manifest)) + + creator.create(package) + + mock_table.set.assert_called_with('key_a', [('field_1', 'value_1'), + ('field_2', 'original_value_2')]) + + +def test_feature_registration(mock_sonic_db, manifest): + mock_feature_table = Mock() + mock_feature_table.get = Mock(return_value=(False, ())) + mock_sonic_db.initial_table = Mock(return_value=mock_feature_table) + mock_sonic_db.persistent_table = Mock(return_value=mock_feature_table) + mock_sonic_db.running_table = Mock(return_value=mock_feature_table) + feature_registry = FeatureRegistry(mock_sonic_db) + feature_registry.register(manifest) + mock_feature_table.set.assert_called_with('test', [ + ('state', 'disabled'), + ('auto_restart', 'enabled'), + ('high_mem_alert', 'disabled'), + ('set_owner', 'local'), + ('has_per_asic_scope', 'False'), + ('has_global_scope', 'True'), + ('has_timer', 'False'), + ]) + + +def test_feature_registration_with_timer(mock_sonic_db, manifest): + manifest['service']['delayed'] = True + mock_feature_table = Mock() + mock_feature_table.get = Mock(return_value=(False, ())) + mock_sonic_db.initial_table = Mock(return_value=mock_feature_table) + mock_sonic_db.persistent_table = Mock(return_value=mock_feature_table) + mock_sonic_db.running_table = Mock(return_value=mock_feature_table) + feature_registry = FeatureRegistry(mock_sonic_db) + feature_registry.register(manifest) + mock_feature_table.set.assert_called_with('test', [ + ('state', 'disabled'), + ('auto_restart', 'enabled'), + ('high_mem_alert', 'disabled'), + ('set_owner', 'local'), + ('has_per_asic_scope', 'False'), + ('has_global_scope', 'True'), + ('has_timer', 'True'), + ]) + + +def test_feature_registration_with_non_default_owner(mock_sonic_db, manifest): + mock_feature_table = Mock() + mock_feature_table.get = Mock(return_value=(False, ())) + mock_sonic_db.initial_table = Mock(return_value=mock_feature_table) + mock_sonic_db.persistent_table = Mock(return_value=mock_feature_table) + mock_sonic_db.running_table = Mock(return_value=mock_feature_table) + feature_registry = FeatureRegistry(mock_sonic_db) + feature_registry.register(manifest, owner='kube') + mock_feature_table.set.assert_called_with('test', [ + ('state', 'disabled'), + ('auto_restart', 'enabled'), + ('high_mem_alert', 'disabled'), + ('set_owner', 'kube'), + ('has_per_asic_scope', 'False'), + ('has_global_scope', 'True'), + ('has_timer', 'False'), + ]) diff --git a/tests/sonic_package_manager/test_utils.py b/tests/sonic_package_manager/test_utils.py new file mode 100644 index 0000000000..c4d8b15840 --- /dev/null +++ b/tests/sonic_package_manager/test_utils.py @@ -0,0 +1,8 @@ +#!/usr/bin/env python + +from sonic_package_manager import utils + + +def test_make_python_identifier(): + assert utils.make_python_identifier('-some-package name').isidentifier() + assert utils.make_python_identifier('01 leading digit').isidentifier() From 06dfd3002166ed072ab345949f57f5e24e21e7a0 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Tue, 23 Mar 2021 14:30:27 +0200 Subject: [PATCH 009/173] add help for some options Signed-off-by: Stepan Blyschak --- sonic_package_manager/main.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sonic_package_manager/main.py b/sonic_package_manager/main.py index dcc048079c..af0a11d60a 100644 --- a/sonic_package_manager/main.py +++ b/sonic_package_manager/main.py @@ -91,7 +91,7 @@ def handle_parse_result(self, ctx, opts, args): PACKAGE_SOURCE_OPTIONS = [ click.option('--from-repository', - help='Install package directly from image registry repository', + help='Fetch package directly from image registry repository', cls=MutuallyExclusiveOption, mutually_exclusive=['from_tarball', 'package_expr']), click.option('--from-tarball', @@ -99,7 +99,7 @@ def handle_parse_result(self, ctx, opts, args): readable=True, file_okay=True, dir_okay=False), - help='Install package from saved image tarball', + help='Fetch package from saved image tarball', cls=MutuallyExclusiveOption, mutually_exclusive=['from_repository', 'package_expr']), click.argument('package-expr', @@ -281,8 +281,8 @@ def changelog(ctx, @repository.command() @click.argument('name', type=str) @click.argument('repository', type=str) -@click.option('--default-reference', type=str) -@click.option('--description', type=str) +@click.option('--default-reference', type=str, help='Default installation reference.') +@click.option('--description', type=str, help='Default installation reference.') @click.pass_context @root_privileges_required def add(ctx, name, repository, default_reference, description): From 725210d908106e45ded934969420bfc8e163e942 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Tue, 23 Mar 2021 14:30:43 +0200 Subject: [PATCH 010/173] add command line reference for sonic-package-manager Signed-off-by: Stepan Blyschak --- doc/Command-Reference.md | 294 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 293 insertions(+), 1 deletion(-) diff --git a/doc/Command-Reference.md b/doc/Command-Reference.md index ffbc0c26f4..b55e36d53d 100644 --- a/doc/Command-Reference.md +++ b/doc/Command-Reference.md @@ -143,6 +143,7 @@ * [Watermark Show commands](#watermark-show-commands) * [Watermark Config commands](#watermark-config-commands) * [Software Installation and Management](#software-installation-and-management) + * [SONiC Package Manager](#sonic-package-manager) * [SONiC Installer](#sonic-installer) * [Troubleshooting Commands](#troubleshooting-commands) * [Routing Stack](#routing-stack) @@ -7961,8 +7962,292 @@ Go Back To [Beginning of the document](#) or [Beginning of this section](#waterm ## Software Installation and Management -SONiC software can be installed in two methods, viz, "using sonic-installer tool", "ONIE Installer". +SONiC software image can be installed in two methods, viz, "using sonic-installer tool", "ONIE Installer". +SONiC feature Docker images (aka "SONiC packages") available to be installed with *sonic-package-manager* utility. + +### SONiC Package Manager + +This is a command line tool that provides functionality to manage SONiC Packages on SONiC device. + +**sonic-package-manager list** + +This command lists all available SONiC packages, their desription, installed version and installation status. +SONiC package status can be *Installed*, *Not installed* or *Built-In*. "Built-In" status means that a feature is built-in to SONiC image and can't be upgraded or uninstalled. + +- Usage: + ``` + sonic-package-manager list + ``` + +- Example: + ``` + admin@sonic:~$ sonic-package-manager list + Name Repository Description Version Status + -------------- --------------------------- ---------------------------- --------- --------- + database docker-database SONiC database package 1.0.0 Built-In + dhcp-relay docker-dhcp-relay SONiC dhcp-relay package 1.0.0 Installed + fpm-frr docker-fpm-frr SONiC fpm-frr package 1.0.0 Built-In + lldp docker-lldp SONiC lldp package 1.0.0 Built-In + macsec docker-macsec SONiC macsec package 1.0.0 Built-In + mgmt-framework docker-sonic-mgmt-framework SONiC mgmt-framework package 1.0.0 Built-In + nat docker-nat SONiC nat package 1.0.0 Built-In + pmon docker-platform-monitor SONiC pmon package 1.0.0 Built-In + radv docker-router-advertiser SONiC radv package 1.0.0 Built-In + sflow docker-sflow SONiC sflow package 1.0.0 Built-In + snmp docker-snmp SONiC snmp package 1.0.0 Built-In + swss docker-orchagent SONiC swss package 1.0.0 Built-In + syncd docker-syncd-mlnx SONiC syncd package 1.0.0 Built-In + teamd docker-teamd SONiC teamd package 1.0.0 Built-In + telemetry docker-sonic-telemetry SONiC telemetry package 1.0.0 Built-In + ``` + +**sonic-package-manager repository add** + +This command will add a new entry in the package database. The package has to be *Not Installed* in order to be removed from package database. *NOTE*: this command requires elevated (root) privileges to run. + +- Usage: + ``` + Usage: sonic-package-manager repository add [OPTIONS] NAME REPOSITORY + + Add a new repository to database. + + Options: + --default-reference TEXT Default installation reference. + --description TEXT Optional package entry description. + --help Show this message and exit. + ``` +- Example: + ``` + admin@sonic:~$ sudo sonic-package-manager repository add \ + cpu-report azure/sonic-cpu-report --default-reference 1.0.0 + ``` + +**sonic-package-manager repository remove** + +This command will remove an entry from the package database. *NOTE*: this command requires elevated (root) privileges to run. + +- Usage: + ``` + Usage: sonic-package-manager repository remove [OPTIONS] NAME + + Remove package from database. + + Options: + --help Show this message and exit. + ``` +- Example: + ``` + admin@sonic:~$ sudo sonic-package-manager repository remove cpu-report + ``` + +**sonic-package-manager install** + +This command pulls and installs package on SONiC host. *NOTE*: this command requires elevated (root) privileges to run. + +- Usage: + ``` + Usage: sonic-package-manager install [OPTIONS] [PACKAGE_EXPR] + + Install package + + Options: + --enable Set the default state of the feature to + enabled and enable feature right after + installation. NOTE: user needs to execute + "config save -y" to make this setting + persistent + + --default-owner [local|kube] Default owner configuration setting for a + feature + + --from-repository TEXT Fetch package directly from image registry + repository NOTE: This argument is mutually + exclusive with arguments: [from_tarball, + package_expr]. + + --from-tarball FILE Fetch package from saved image tarball + NOTE: This argument is mutually exclusive + with arguments: [from_repository, + package_expr]. + + -f, --force Force operation by ignoring failures + -y, --yes Automatically answer yes on prompts + -v, --verbosity LVL Either CRITICAL, ERROR, WARNING, INFO or + DEBUG + + --skip-cli-plugin-installation Do not install CLI plugins provided by the + package on the host OS. NOTE: In case when + package /cli/mandatory field is set to True + this option will fail the installation. + + --help Show this message and exit. + ``` +- Example: + ``` + admin@sonic:~$ sudo sonic-package-manager install dhcp-relay==1.0.2 + ``` + ``` + admin@sonic:~$ sudo sonic-package-manager install --from-repository azure/sonic-cpu-report:latest + ``` + ``` + admin@sonic:~$ sudo sonic-package-manager install --from-tarball sonic-docker-image.gz + ``` + +**sonic-package-manager uninstall** + +This command uninstalls package from SONiC host. *NOTE*: this command requires elevated (root) privileges to run. + +- Usage: + ``` + Usage: sonic-package-manager uninstall [OPTIONS] NAME + + Uninstall package + + Options: + -f, --force Force operation by ignoring failures + -y, --yes Automatically answer yes on prompts + -v, --verbosity LVL Either CRITICAL, ERROR, WARNING, INFO or DEBUG + --help Show this message and exit. + + ``` +- Example: + ``` + admin@sonic:~$ sudo sonic-package-manager uninstall dhcp-relay + ``` + +**sonic-package-manager upgrade** + +This command upgrades package on SONiC host to a newer version. The procedure of upgrading a package will restart the corresponding service. *NOTE*: this command requires elevated (root) privileges to run. + +- Usage: + ``` + Usage: sonic-package-manager upgrade [OPTIONS] [PACKAGE_EXPR] + + Upgrade package + + Options: + --from-repository TEXT Fetch package directly from image registry + repository NOTE: This argument is mutually + exclusive with arguments: [package_expr, + from_tarball]. + + --from-tarball FILE Fetch package from saved image tarball + NOTE: This argument is mutually exclusive + with arguments: [package_expr, + from_repository]. + + -f, --force Force operation by ignoring failures + -y, --yes Automatically answer yes on prompts + -v, --verbosity LVL Either CRITICAL, ERROR, WARNING, INFO or + DEBUG + + --skip-cli-plugin-installation Do not install CLI plugins provided by the + package on the host OS. NOTE: In case when + package /cli/mandatory field is set to True + this option will fail the installation. + + --help Show this message and exit. + ``` +- Example: + ``` + admin@sonic:~$ sudo sonic-package-manager upgrade dhcp-relay==2.0.0 + ``` + ``` + admin@sonic:~$ sudo sonic-package-manager upgrade --from-repository azure/sonic-cpu-report:latest + ``` + ``` + admin@sonic:~$ sudo sonic-package-manager upgrade --from-tarball sonic-docker-image.gz + ``` + +**sonic-package-manager show package versions** + +This command will access repository for corresponding package and retrieve a list of available versions. + +- Usage: + ``` + Usage: sonic-package-manager show package versions [OPTIONS] NAME + + Print available versions + + Options: + --all Show all available tags in repository + --plain Plain output + --help Show this message and exit. + ``` +- Example: + ``` + admin@sonic:~$ sonic-package-manager show package versions dhcp-relay + • 1.0.0 + • 1.0.2 + • 2.0.0 + ``` + +**sonic-package-manager show package changelog** + +This command fetches the changelog from package manifest and displays it. *NOTE*: package changelog can be retrieved from registry or read from image tarball without installing it. + +- Usage: + ``` + Usage: sonic-package-manager show package changelog [OPTIONS] [PACKAGE_EXPR] + + Print package changelog + + Options: + --from-repository TEXT Fetch package directly from image registry + repository NOTE: This argument is mutually exclusive + with arguments: [from_tarball, package_expr]. + --from-tarball FILE Fetch package from saved image tarball NOTE: This + argument is mutually exclusive with arguments: + [package_expr, from_repository]. + --help Show this message and exit. + ``` +- Example: + ``` + admin@sonic:~$ sonic-package-manager show package changelog dhcp-relay + 1.0.0: + + • Initial release + + Author (author@email.com) Mon, 25 May 2020 12:25:00 +0300 + ``` + +**sonic-package-manager show package manifest** + +This command fetches the package manifest and displays it. *NOTE*: package manifest can be retrieved from registry or read from image tarball without installing it. + +- Usage: + ``` + Usage: sonic-package-manager show package manifest [OPTIONS] [PACKAGE_EXPR] + + Print package manifest content + + Options: + --from-repository TEXT Fetch package directly from image registry + repository NOTE: This argument is mutually exclusive + with arguments: [package_expr, from_tarball]. + --from-tarball FILE Fetch package from saved image tarball NOTE: This + argument is mutually exclusive with arguments: + [from_repository, package_expr]. + -v, --verbosity LVL Either CRITICAL, ERROR, WARNING, INFO or DEBUG + --help Show this message and exit. + ``` +- Example: + ``` + admin@sonic:~$ sonic-package-manager show package manifest dhcp-relay==2.0.0 + { + "version": "1.0.0", + "package": { + "version": "2.0.0", + "depends": [ + "database>=1.0.0,<2.0.0" + ] + }, + "service": { + "name": "dhcp_relay" + } + } + ``` ### SONiC Installer This is a command line tool available as part of the SONiC software; If the device is already running the SONiC software, this tool can be used to install an alternate image in the partition. @@ -8033,6 +8318,13 @@ This command is used to install a new image on the alternate image partition. T Done ``` +SONiC image installation will install SONiC packages that are installed in currently running SONiC image. In order to perform clean SONiC installation use *--skip-package-migration* option when installing SONiC image: + +- Example: + ``` + admin@sonic:~$ sudo sonic-installer install https://sonic-jenkins.westus.cloudapp.azure.com/job/xxxx/job/buildimage-xxxx-all/xxx/artifact/target/sonic-xxxx.bin --skip-package-migration + ``` + **sonic-installer set_default** This command is be used to change the image which can be loaded by default in all the subsequent reboots. From f2ee39f27e16484defac5ca8af5fcd0dbbf5e009 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Tue, 23 Mar 2021 15:35:42 +0200 Subject: [PATCH 011/173] fix upgrade command missing argument Signed-off-by: Stepan Blyschak --- sonic_package_manager/main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sonic_package_manager/main.py b/sonic_package_manager/main.py index af0a11d60a..713113da61 100644 --- a/sonic_package_manager/main.py +++ b/sonic_package_manager/main.py @@ -378,7 +378,8 @@ def upgrade(ctx, from_repository, from_tarball, force, - yes): + yes, + skip_cli_plugin_installation): """ Upgrade package """ manager: PackageManager = ctx.obj From 7bf937ed5160bd08bca79fd5151c92e0771a2651 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Thu, 25 Mar 2021 14:00:08 +0200 Subject: [PATCH 012/173] warm-reboot support for sonic packages Signed-off-by: Stepan Blyschak --- config/main.py | 12 +- scripts/fast-reboot | 119 ++++++++---------- scripts/generate_shutdown_order.py | 15 +++ setup.py | 2 + sonic_package_manager/manager.py | 23 +++- sonic_package_manager/manifest.py | 13 +- .../service_creator/creator.py | 63 +++++++++- tests/sonic_package_manager/conftest.py | 48 ++++++- tests/sonic_package_manager/test_manager.py | 4 +- .../test_service_creator.py | 29 ++++- 10 files changed, 241 insertions(+), 87 deletions(-) create mode 100644 scripts/generate_shutdown_order.py diff --git a/config/main.py b/config/main.py index d27562bd4e..5fb272a807 100644 --- a/config/main.py +++ b/config/main.py @@ -1833,20 +1833,28 @@ def warm_restart(ctx, redis_unix_socket_path): ctx.obj = {'db': config_db, 'state_db': state_db, 'prefix': prefix} @warm_restart.command('enable') -@click.argument('module', metavar='', default='system', required=False, type=click.Choice(["system", "swss", "bgp", "teamd"])) +@click.argument('module', metavar='', default='system', required=False) @click.pass_context def warm_restart_enable(ctx, module): state_db = ctx.obj['state_db'] + config_db = ctx.obj['db'] + feature_table = config_db.get_table('FEATURE') + if module != 'system' and module not in feature_table: + exit('Feature {} is unknown'.format(module)) prefix = ctx.obj['prefix'] _hash = '{}{}'.format(prefix, module) state_db.set(state_db.STATE_DB, _hash, 'enable', 'true') state_db.close(state_db.STATE_DB) @warm_restart.command('disable') -@click.argument('module', metavar='', default='system', required=False, type=click.Choice(["system", "swss", "bgp", "teamd"])) +@click.argument('module', metavar='', default='system', required=False) @click.pass_context def warm_restart_enable(ctx, module): state_db = ctx.obj['state_db'] + config_db = ctx.obj['db'] + feature_table = config_db.get_table('FEATURE') + if module != 'system' and module not in feature_table: + exit('Feature {} is unknown'.format(module)) prefix = ctx.obj['prefix'] _hash = '{}{}'.format(prefix, module) state_db.set(state_db.STATE_DB, _hash, 'enable', 'false') diff --git a/scripts/fast-reboot b/scripts/fast-reboot index 92648bd207..d11b143d56 100755 --- a/scripts/fast-reboot +++ b/scripts/fast-reboot @@ -7,6 +7,7 @@ WARM_DIR=/host/warmboot REDIS_FILE=dump.rdb REBOOT_SCRIPT_NAME=$(basename $0) REBOOT_TYPE="${REBOOT_SCRIPT_NAME}" +SHUTDOWN_ORDER_FILE="/etc/sonic/${REBOOT_TYPE}_order" VERBOSE=no FORCE=no IGNORE_ASIC=no @@ -547,82 +548,72 @@ if [ -x ${LOG_SSD_HEALTH} ]; then fi -# Kill nat docker after saving the conntrack table -debug "Stopping nat ..." -/usr/local/bin/dump_nat_entries.py -docker kill nat > /dev/null || true -systemctl stop nat -debug "Stopped nat ..." - -# Kill radv before stopping BGP service to prevent announcing our departure. -debug "Stopping radv service..." -systemctl stop radv -debug "Stopped radv service..." - -# Kill bgpd to start the bgp graceful restart procedure -debug "Stopping bgp ..." -systemctl stop bgp -debug "Stopped bgp ..." - -# Kill sflow docker -debug "Stopping sflow ..." -container kill sflow &> /dev/null || debug "Docker sflow is not running ($?) ..." -systemctl stop sflow -debug "Stopped sflow ..." - -# Kill lldp, otherwise it sends informotion about reboot. -# We call `docker kill lldp` to ensure the container stops as quickly as possible, -# then immediately call `systemctl stop lldp` to prevent the service from -# restarting the container automatically. -container kill lldp &> /dev/null || debug "Docker lldp is not running ($?) ..." -systemctl stop lldp - -if [[ "$REBOOT_TYPE" = "fast-reboot" ]]; then - debug "Stopping teamd ..." - systemctl stop teamd - debug "Stopped teamd ..." +if [[ -f ${SHUTDOWN_ORDER_FILE} ]]; then + SERVICE_TO_STOP="$(cat ${SHUTDOWN_ORDER_FILE})" +else + # TODO: to be removed once sonic-buildimage change is in + if [[ "${REBOOT_TYPE}" == "fast-reboot" ]]; then + SERVICE_TO_STOP="nat radv bgp sflow lldp swss teamd syncd" + elif [[ "${REBOOT_TYPE}" == "fastfast-reboot" || "${REBOOT_TYPE}" == "warm-reboot" ]]; then + SERVICE_TO_STOP="nat radv bgp sflow lldp teamd swss syncd" + else + error "Unexpected reboot type ${REBOOT_TYPE}" + exit $EXIT_FAILURE + fi fi -debug "Stopping swss service ..." -systemctl stop swss -debug "Stopped swss service ..." +for service in ${SERVICE_TO_STOP}; do + debug "Stopping ${service} ..." -if [[ "$REBOOT_TYPE" = "warm-reboot" || "$REBOOT_TYPE" = "fastfast-reboot" ]]; then - # Pre-shutdown syncd - initialize_pre_shutdown + # TODO: These exceptions for nat, sflow, lldp + # have to be coded in corresponding service scripts - if [[ "x$sonic_asic_type" == x"mellanox" ]]; then - check_issu_bank_file + if [[ "${service}" = "nat" ]]; then + /usr/local/bin/dump_nat_entries.py fi - request_pre_shutdown - - wait_for_pre_shutdown_complete_or_fail - - if [[ "x$sonic_asic_type" == x"mellanox" ]]; then - check_issu_bank_file + if [[ "${service}" = "nat" || "${service}" = "sflow" || "${service}" = "lldp" ]]; then + container kill "${service}" &> /dev/null || debug "Docker ${service} is not running ($?) ..." fi - # Warm reboot: dump state to host disk - if [[ "$REBOOT_TYPE" = "fastfast-reboot" ]]; then - sonic-db-cli ASIC_DB FLUSHDB > /dev/null - sonic-db-cli COUNTERS_DB FLUSHDB > /dev/null - sonic-db-cli FLEX_COUNTER_DB FLUSHDB > /dev/null + if [[ "${service}" = "syncd" ]]; then + systemctl stop ${service} || debug "Ignore stopping ${service} service error $?" + else + systemctl stop ${service} fi - # TODO: backup_database preserves FDB_TABLE - # need to cleanup as well for fastfast boot case - backup_database + debug "Stopped ${service}" - # Stop teamd gracefully - debug "Stopping teamd ..." - systemctl stop teamd - debug "Stopped teamd ..." -fi + if [[ "${service}" = "swss" ]]; then + if [[ "$REBOOT_TYPE" = "warm-reboot" || "$REBOOT_TYPE" = "fastfast-reboot" ]]; then + # Pre-shutdown syncd + initialize_pre_shutdown + + if [[ "x$sonic_asic_type" == x"mellanox" ]]; then + check_issu_bank_file + fi -debug "Stopping syncd ..." -systemctl stop syncd || debug "Ignore stopping syncd service error $?" -debug "Stopped syncd ..." + request_pre_shutdown + + wait_for_pre_shutdown_complete_or_fail + + if [[ "x$sonic_asic_type" == x"mellanox" ]]; then + check_issu_bank_file + fi + + # Warm reboot: dump state to host disk + if [[ "$REBOOT_TYPE" = "fastfast-reboot" ]]; then + sonic-db-cli ASIC_DB FLUSHDB > /dev/null + sonic-db-cli COUNTERS_DB FLUSHDB > /dev/null + sonic-db-cli FLEX_COUNTER_DB FLUSHDB > /dev/null + fi + + # TODO: backup_database preserves FDB_TABLE + # need to cleanup as well for fastfast boot case + backup_database + fi + fi +done # Kill other containers to make the reboot faster # We call `docker kill ...` to ensure the container stops as quickly as possible, diff --git a/scripts/generate_shutdown_order.py b/scripts/generate_shutdown_order.py new file mode 100644 index 0000000000..a2427b3691 --- /dev/null +++ b/scripts/generate_shutdown_order.py @@ -0,0 +1,15 @@ +#!/usr/bin/python3 + +''' This script is used to generate initial warm/fast shutdown order file ''' + +from sonic_package_manager import PackageManager + +def main(): + manager = PackageManager.get_manager() + installed_packages = manager.get_installed_packages().values() + print('installed packages {}'.format(installed_packages)) + manager.service_creator.generate_shutdown_sequence_files(installed_packages) + print('Done.') + +if __name__ == '__main__': + main() diff --git a/setup.py b/setup.py index 9847435671..c93ff6d48d 100644 --- a/setup.py +++ b/setup.py @@ -90,6 +90,7 @@ 'scripts/fdbshow', 'scripts/gearboxutil', 'scripts/generate_dump', + 'scripts/generate_shutdown_order.py', 'scripts/intfutil', 'scripts/intfstat', 'scripts/ipintutil', @@ -178,6 +179,7 @@ 'sonic-yang-mgmt', 'swsssdk>=2.0.1', 'tabulate==0.8.2', + 'toposort==1.6', 'www-authenticate==0.9.2', 'xmltodict==0.12.0', ], diff --git a/sonic_package_manager/manager.py b/sonic_package_manager/manager.py index f1643a1846..3c128c1a36 100644 --- a/sonic_package_manager/manager.py +++ b/sonic_package_manager/manager.py @@ -344,8 +344,13 @@ def install_from_source(self, source.install(package) exit_stack.callback(rollback_wrapper(source.uninstall, package)) - self.service_creator.create(package, state=feature_state, owner=default_owner) - exit_stack.callback(rollback_wrapper(self.service_creator.remove, package)) + self.service_creator.create(package, + installed_packages, + state=feature_state, + owner=default_owner) + exit_stack.callback(rollback_wrapper(self.service_creator.remove, + package, + installed_packages)) if not skip_cli_plugin_installation: self._install_cli_plugins(package) @@ -399,7 +404,7 @@ def uninstall(self, name: str, force=False): try: self._uninstall_cli_plugins(package) - self.service_creator.remove(package) + self.service_creator.remove(package, installed_packages) # Clean containers based on this image containers = self.docker.ps(filters={'ancestor': package.image_id}, all=True) @@ -509,9 +514,13 @@ def upgrade_from_source(self, exit_stack.callback(rollback_wrapper(self._systemctl_action, old_package, 'start')) - self.service_creator.remove(old_package, deregister_feature=False) + self.service_creator.remove(old_package, + installed_packages, + deregister_feature=False) exit_stack.callback(rollback_wrapper(self.service_creator.create, - old_package, register_feature=False)) + old_package, + installed_packages, + register_feature=False)) # This is no return point, after we start removing old Docker images # there is no guaranty we can actually successfully roll-back. @@ -523,7 +532,9 @@ def upgrade_from_source(self, self.docker.rmi(old_package.image_id, force=True) - self.service_creator.create(new_package, register_feature=False) + self.service_creator.create(new_package, + installed_packages, + register_feature=False) if self.feature_registry.is_feature_enabled(new_feature): self._systemctl_action(new_package, 'start') diff --git a/sonic_package_manager/manifest.py b/sonic_package_manager/manifest.py index 74da06a956..5a55c87e98 100644 --- a/sonic_package_manager/manifest.py +++ b/sonic_package_manager/manifest.py @@ -171,6 +171,14 @@ def unmarshal(self, value): ManifestField('asic-service', DefaultMarshaller(bool), False), ManifestField('host-service', DefaultMarshaller(bool), True), ManifestField('delayed', DefaultMarshaller(bool), False), + ManifestRoot('warm-shutdown', [ + ManifestArray('after', DefaultMarshaller(str)), + ManifestArray('before', DefaultMarshaller(str)), + ]), + ManifestRoot('fast-shutdown', [ + ManifestArray('after', DefaultMarshaller(str)), + ManifestArray('before', DefaultMarshaller(str)), + ]), ]), ManifestRoot('container', [ ManifestField('privileged', DefaultMarshaller(bool), False), @@ -184,9 +192,10 @@ def unmarshal(self, value): ManifestArray('tmpfs', DefaultMarshaller(str)), ]), ManifestArray('processes', ManifestRoot('processes', [ - ManifestField('critical', DefaultMarshaller(bool)), + ManifestField('critical', DefaultMarshaller(bool), False), ManifestField('name', DefaultMarshaller(str)), - ManifestField('command', DefaultMarshaller(str)), + ManifestField('command', DefaultMarshaller(str), ''), + ManifestField('reconciles', DefaultMarshaller(bool), False), ])), ManifestRoot('cli', [ ManifestField('mandatory', DefaultMarshaller(bool), False), diff --git a/sonic_package_manager/service_creator/creator.py b/sonic_package_manager/service_creator/creator.py index f62d0a3074..978d53bf0d 100644 --- a/sonic_package_manager/service_creator/creator.py +++ b/sonic_package_manager/service_creator/creator.py @@ -3,16 +3,17 @@ import os import stat import subprocess -from typing import Dict +from collections import defaultdict +from typing import Dict, List import jinja2 as jinja2 from prettyprinter import pformat - from sonic_package_manager.logger import log from sonic_package_manager.package import Package from sonic_package_manager.service_creator import ETC_SONIC_PATH from sonic_package_manager.service_creator.feature import FeatureRegistry from sonic_package_manager.service_creator.utils import in_chroot +from toposort import toposort_flatten, CircularDependencyError SERVICE_FILE_TEMPLATE = 'sonic.service.j2' TIMER_UNIT_TEMPLATE = 'timer.unit.j2' @@ -108,9 +109,11 @@ def __init__(self, feature_registry: FeatureRegistry, sonic_db): def create(self, package: Package, + all_packages: List[Package] = None, register_feature=True, state='enabled', owner='local'): + all_packages = all_packages or [] try: self.generate_container_mgmt(package) self.generate_service_mgmt(package) @@ -118,10 +121,11 @@ def create(self, self.generate_systemd_service(package) self.generate_monit_conf(package) self.generate_dump_script(package) + self.generate_service_reconciliation_file(package) self.set_initial_config(package) - self.post_operation_hook() + self.post_operation_hook(all_packages) if register_feature: self.feature_registry.register(package.manifest, @@ -130,7 +134,11 @@ def create(self, self.remove(package, not register_feature) raise - def remove(self, package: Package, deregister_feature=True): + def remove(self, + package: Package, + all_packages: List[Package] = None, + deregister_feature=True): + all_packages = all_packages or [] name = package.manifest['service']['name'] def remove_file(path): @@ -144,18 +152,21 @@ def remove_file(path): remove_file(os.path.join(SERVICE_MGMT_SCRIPT_LOCATION, f'{name}.sh')) remove_file(os.path.join(DOCKER_CTL_SCRIPT_LOCATION, f'{name}.sh')) remove_file(os.path.join(DEBUG_DUMP_SCRIPT_LOCATION, f'{name}')) + remove_file(os.path.join(ETC_SONIC_PATH, f'{name}_reconcile')) self.update_dependent_list_file(package, remove=True) - self.post_operation_hook() + self.post_operation_hook(all_packages) if deregister_feature: self.feature_registry.deregister(package.manifest['service']['name']) - def post_operation_hook(self): + def post_operation_hook(self, all_packages: List[Package]): if not in_chroot(): run_command('systemctl daemon-reload') run_command('systemctl reload monit') + + self.generate_shutdown_sequence_files(all_packages) def generate_container_mgmt(self, package: Package): image_id = package.image_id @@ -303,6 +314,46 @@ def generate_dump_script(self, package): render_template(scrip_template, script_path, render_ctx, executable=True) log.info(f'generated {script_path}') + def generate_shutdown_sequence(self, installed_packages, reboot_type): + shutdown_graph = dict() + for package in installed_packages: + after = set(package.manifest['service'][f'{reboot_type}-shutdown']['after']) + before = set(package.manifest['service'][f'{reboot_type}-shutdown']['before']) + if not after and not before: + continue + shutdown_graph.setdefault(package.name, set()) + shutdown_graph[package.name].update(after) + + for service in before: + shutdown_graph.setdefault(service, set()) + shutdown_graph[service].update({package.name}) + + log.debug(f'shutdown graph {pformat(shutdown_graph)}') + + try: + order = toposort_flatten(shutdown_graph) + except CircularDependencyError as err: + raise ServiceCreatorError(f'Circular dependency found in {reboot_type} shutdown graph: {err}') + + log.debug(f'shutdown order {pformat(order)}') + return order + + def generate_shutdown_sequence_file(self, installed_packages, reboot_type): + order = self.generate_shutdown_sequence(installed_packages, reboot_type) + with open(os.path.join(ETC_SONIC_PATH, f'{reboot_type}-reboot_order'), 'w') as file: + file.write(' '.join(order)) + + def generate_shutdown_sequence_files(self, installed_packages): + for reboot_type in ('fast', 'warm'): + self.generate_shutdown_sequence_file(installed_packages, reboot_type) + + def generate_service_reconciliation_file(self, package): + name = package.manifest['service']['name'] + processes = [process['name'] for process in package.manifest['processes'] + if process['reconciles']] + with open(os.path.join(ETC_SONIC_PATH, f'{name}_reconcile'), 'w') as file: + file.write(' '.join(processes)) + def set_initial_config(self, package): init_cfg = package.manifest['package']['init-cfg'] diff --git a/tests/sonic_package_manager/conftest.py b/tests/sonic_package_manager/conftest.py index bf787bdab4..fc8005212d 100644 --- a/tests/sonic_package_manager/conftest.py +++ b/tests/sonic_package_manager/conftest.py @@ -75,6 +75,35 @@ def __init__(self): components={ 'libswsscommon': Version.parse('1.0.0'), 'libsairedis': Version.parse('1.0.0') + }, + warm_shutdown={ + 'before': ['syncd'], + }, + fast_shutdown={ + 'before': ['syncd'], + }, + processes=[ + { + 'name': 'orchagent', + 'reconciles': True, + }, + { + 'name': 'neighsyncd', + 'reconciles': True, + } + ], + ) + self.add('docker-teamd', 'latest', 'teamd', '1.0.0', + components={ + 'libswsscommon': Version.parse('1.0.0'), + 'libsairedis': Version.parse('1.0.0') + }, + warm_shutdown={ + 'before': ['syncd'], + 'after': ['swss'], + }, + fast_shutdown={ + 'before': ['swss'], } ) self.add('Azure/docker-test', '1.6.0', 'test-package', '1.6.0') @@ -108,7 +137,9 @@ def from_tarball(self, filepath: str) -> Manifest: components = self.metadata_store[path][ref]['components'] return Metadata(manifest, components) - def add(self, repo, reference, name, version, components=None): + def add(self, repo, reference, name, version, components=None, + warm_shutdown=None, fast_shutdown=None, + processes=None): repo_dict = self.metadata_store.setdefault(repo, {}) repo_dict[reference] = { 'manifest': { @@ -119,7 +150,10 @@ def add(self, repo, reference, name, version, components=None): }, 'service': { 'name': name, - } + 'warm-shutdown': warm_shutdown or {}, + 'fast-shutdown': fast_shutdown or {}, + }, + 'processes': processes or {} }, 'components': components or {}, } @@ -189,6 +223,16 @@ def fake_db(fake_metadata_resolver): installed=True, built_in=True ) + add_package( + content, + fake_metadata_resolver, + 'docker-teamd', + 'latest', + description='SONiC teamd service', + default_reference='1.0.0', + installed=True, + built_in=True + ) add_package( content, fake_metadata_resolver, diff --git a/tests/sonic_package_manager/test_manager.py b/tests/sonic_package_manager/test_manager.py index 256832118e..ee0b817624 100644 --- a/tests/sonic_package_manager/test_manager.py +++ b/tests/sonic_package_manager/test_manager.py @@ -250,12 +250,12 @@ def test_upgrade_from_file_known_package(package_manager, fake_db, sonic_fs): def test_installation_non_default_owner(package_manager, anything, mock_service_creator): package_manager.install('test-package', default_owner='kube') - mock_service_creator.create.assert_called_once_with(anything, state='disabled', owner='kube') + mock_service_creator.create.assert_called_once_with(anything, anything, state='disabled', owner='kube') def test_installation_enabled(package_manager, anything, mock_service_creator): package_manager.install('test-package', enable=True) - mock_service_creator.create.assert_called_once_with(anything, state='enabled', owner='local') + mock_service_creator.create.assert_called_once_with(anything, anything, state='enabled', owner='local') def test_installation_fault(package_manager, mock_docker_api, mock_service_creator): diff --git a/tests/sonic_package_manager/test_service_creator.py b/tests/sonic_package_manager/test_service_creator.py index e69b4447f8..006b1af4ea 100644 --- a/tests/sonic_package_manager/test_service_creator.py +++ b/tests/sonic_package_manager/test_service_creator.py @@ -37,15 +37,30 @@ def manifest(): 'volumes': [ '/etc/sonic:/etc/sonic:ro' ] - } + }, + 'processes': [ + { + 'name': 'test-process', + 'reconciles': True, + }, + { + 'name': 'test-process-2', + 'reconciles': False, + }, + { + 'name': 'test-process-3', + 'reconciles': True, + }, + ] }) -def test_service_creator(sonic_fs, manifest, mock_feature_registry, mock_sonic_db): +def test_service_creator(sonic_fs, manifest, package_manager, mock_feature_registry, mock_sonic_db): creator = ServiceCreator(mock_feature_registry, mock_sonic_db) entry = PackageEntry('test', 'azure/sonic-test') package = Package(entry, Metadata(manifest)) - creator.create(package) + installed_packages = package_manager.get_installed_packages().values() + creator.create(package, all_packages=installed_packages) assert sonic_fs.exists(os.path.join(ETC_SONIC_PATH, 'swss_dependent')) assert sonic_fs.exists(os.path.join(DOCKER_CTL_SCRIPT_LOCATION, 'test.sh')) @@ -53,6 +68,14 @@ def test_service_creator(sonic_fs, manifest, mock_feature_registry, mock_sonic_d assert sonic_fs.exists(os.path.join(SYSTEMD_LOCATION, 'test.service')) assert sonic_fs.exists(os.path.join(MONIT_CONF_LOCATION, 'monit_test')) + def read_file(name): + with open(os.path.join(ETC_SONIC_PATH, name)) as file: + return file.read() + + assert read_file('warm-reboot_order') == 'swss teamd syncd' + assert read_file('fast-reboot_order') == 'teamd swss syncd' + assert read_file('test_reconcile') == 'test-process test-process-3' + def test_service_creator_with_timer_unit(sonic_fs, manifest, mock_feature_registry, mock_sonic_db): creator = ServiceCreator(mock_feature_registry, mock_sonic_db) From c7245694eef3c33a12c71ef137a33c2cf90e284c Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Thu, 25 Mar 2021 15:14:51 +0200 Subject: [PATCH 013/173] iteritems -> items Signed-off-by: Stepan Blyschak --- sonic_package_manager/metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sonic_package_manager/metadata.py b/sonic_package_manager/metadata.py index e5f9dbb3a5..99246f1c6a 100644 --- a/sonic_package_manager/metadata.py +++ b/sonic_package_manager/metadata.py @@ -22,7 +22,7 @@ def deep_update(dst: Dict, src: Dict) -> Dict: New merged dictionary. """ - for key, value in src.iteritems(): + for key, value in src.items(): if isinstance(value, dict): node = dst.setdefault(key, {}) deep_update(node, value) From 3dc793fa8f81ead54c37b183dc5ac76c3618a0a2 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Thu, 25 Mar 2021 16:06:58 +0200 Subject: [PATCH 014/173] fix issue in metadata dict generation Signed-off-by: Stepan Blyschak --- sonic_package_manager/metadata.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sonic_package_manager/metadata.py b/sonic_package_manager/metadata.py index 99246f1c6a..7f7c25ceaf 100644 --- a/sonic_package_manager/metadata.py +++ b/sonic_package_manager/metadata.py @@ -61,8 +61,8 @@ def translate_plain_to_tree(plain: Dict[str, str], sep='.') -> Dict: res[key] = value continue namespace, key = key.split(sep, 1) - res.setdefault(key, {}) - deep_update(res[key], translate_plain_to_tree({key: value})) + res.setdefault(namespace, {}) + deep_update(res[namespace], translate_plain_to_tree({key: value})) return res From ea148672609fc719a6fe0388741e07bf9f44249a Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Thu, 25 Mar 2021 17:16:37 +0200 Subject: [PATCH 015/173] fix creator with service name Signed-off-by: Stepan Blyschak --- sonic_package_manager/service_creator/creator.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sonic_package_manager/service_creator/creator.py b/sonic_package_manager/service_creator/creator.py index 978d53bf0d..1f3a15c80d 100644 --- a/sonic_package_manager/service_creator/creator.py +++ b/sonic_package_manager/service_creator/creator.py @@ -321,12 +321,13 @@ def generate_shutdown_sequence(self, installed_packages, reboot_type): before = set(package.manifest['service'][f'{reboot_type}-shutdown']['before']) if not after and not before: continue + name = package.manifest['service']['name'] shutdown_graph.setdefault(package.name, set()) - shutdown_graph[package.name].update(after) + shutdown_graph[name].update(after) for service in before: shutdown_graph.setdefault(service, set()) - shutdown_graph[service].update({package.name}) + shutdown_graph[service].update({name}) log.debug(f'shutdown graph {pformat(shutdown_graph)}') From faeadcf12b1829d24c9762b01f1d87b320d1909c Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Thu, 25 Mar 2021 17:20:06 +0200 Subject: [PATCH 016/173] default value for component constraint Signed-off-by: Stepan Blyschak --- sonic_package_manager/manifest.py | 2 +- tests/sonic_package_manager/test_manifest.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/sonic_package_manager/manifest.py b/sonic_package_manager/manifest.py index 74da06a956..5a357d668d 100644 --- a/sonic_package_manager/manifest.py +++ b/sonic_package_manager/manifest.py @@ -150,7 +150,7 @@ def unmarshal(self, value): ManifestField('version', ParsedMarshaller(Version)), ManifestField('name', DefaultMarshaller(str)), ManifestField('description', DefaultMarshaller(str), ''), - ManifestField('base-os', ParsedMarshaller(ComponentConstraints), dict()), + ManifestField('base-os', ParsedMarshaller(ComponentConstraints), ComponentConstraints()), ManifestArray('depends', ParsedMarshaller(PackageConstraint)), ManifestArray('breaks', ParsedMarshaller(PackageConstraint)), ManifestField('init-cfg', DefaultMarshaller(dict), dict()), diff --git a/tests/sonic_package_manager/test_manifest.py b/tests/sonic_package_manager/test_manifest.py index f6028e0437..cf3c02b6d1 100644 --- a/tests/sonic_package_manager/test_manifest.py +++ b/tests/sonic_package_manager/test_manifest.py @@ -2,6 +2,7 @@ import pytest +from sonic_package_manager.constraint import ComponentConstraints from sonic_package_manager.manifest import Manifest, ManifestError from sonic_package_manager.version import VersionRange @@ -12,7 +13,7 @@ def test_manifest_v1_defaults(): 'service': {'name': 'test'}}) assert manifest['package']['depends'] == [] assert manifest['package']['breaks'] == [] - assert manifest['package']['base-os'] == dict() + assert manifest['package']['base-os'] == ComponentConstraints() assert not manifest['service']['asic-service'] assert manifest['service']['host-service'] From 31138d051721216d3a8d0ae6cae1f0dbb5803019 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Fri, 26 Mar 2021 11:24:29 +0200 Subject: [PATCH 017/173] fix passing installed_packages Signed-off-by: Stepan Blyschak --- sonic_package_manager/manager.py | 12 ++++++------ sonic_package_manager/service_creator/creator.py | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/sonic_package_manager/manager.py b/sonic_package_manager/manager.py index 3c128c1a36..75390f30cd 100644 --- a/sonic_package_manager/manager.py +++ b/sonic_package_manager/manager.py @@ -345,12 +345,12 @@ def install_from_source(self, exit_stack.callback(rollback_wrapper(source.uninstall, package)) self.service_creator.create(package, - installed_packages, + installed_packages.values(), state=feature_state, owner=default_owner) exit_stack.callback(rollback_wrapper(self.service_creator.remove, package, - installed_packages)) + installed_packages.values())) if not skip_cli_plugin_installation: self._install_cli_plugins(package) @@ -404,7 +404,7 @@ def uninstall(self, name: str, force=False): try: self._uninstall_cli_plugins(package) - self.service_creator.remove(package, installed_packages) + self.service_creator.remove(package, installed_packages.values()) # Clean containers based on this image containers = self.docker.ps(filters={'ancestor': package.image_id}, all=True) @@ -515,11 +515,11 @@ def upgrade_from_source(self, old_package, 'start')) self.service_creator.remove(old_package, - installed_packages, + installed_packages.values(), deregister_feature=False) exit_stack.callback(rollback_wrapper(self.service_creator.create, old_package, - installed_packages, + installed_packages.values(), register_feature=False)) # This is no return point, after we start removing old Docker images @@ -533,7 +533,7 @@ def upgrade_from_source(self, self.docker.rmi(old_package.image_id, force=True) self.service_creator.create(new_package, - installed_packages, + installed_packages.values(), register_feature=False) if self.feature_registry.is_feature_enabled(new_feature): diff --git a/sonic_package_manager/service_creator/creator.py b/sonic_package_manager/service_creator/creator.py index 1f3a15c80d..6299ed3747 100644 --- a/sonic_package_manager/service_creator/creator.py +++ b/sonic_package_manager/service_creator/creator.py @@ -322,7 +322,7 @@ def generate_shutdown_sequence(self, installed_packages, reboot_type): if not after and not before: continue name = package.manifest['service']['name'] - shutdown_graph.setdefault(package.name, set()) + shutdown_graph.setdefault(name, set()) shutdown_graph[name].update(after) for service in before: From ae1ce45604d69ad24627754728a898236447c733 Mon Sep 17 00:00:00 2001 From: Stepan Blyshchak <38952541+stepanblyschak@users.noreply.github.com> Date: Mon, 29 Mar 2021 10:20:57 +0300 Subject: [PATCH 018/173] Update doc/Command-Reference.md Co-authored-by: Joe LeVeque --- doc/Command-Reference.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/Command-Reference.md b/doc/Command-Reference.md index b55e36d53d..36837ee4c2 100644 --- a/doc/Command-Reference.md +++ b/doc/Command-Reference.md @@ -7962,7 +7962,9 @@ Go Back To [Beginning of the document](#) or [Beginning of this section](#waterm ## Software Installation and Management -SONiC software image can be installed in two methods, viz, "using sonic-installer tool", "ONIE Installer". +SONiC image can be installed in one of two methods: +1. From within a running SONiC iamge using the `sonic-installer` utility +2. From the vendor's bootloader (E.g., ONIE, Aboot, etc.) SONiC feature Docker images (aka "SONiC packages") available to be installed with *sonic-package-manager* utility. From 5c045e0c50fbb286fe88881fc21735d59bd5dbb6 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Tue, 30 Mar 2021 12:38:02 +0300 Subject: [PATCH 019/173] use defaultdict Signed-off-by: Stepan Blyschak --- sonic_package_manager/service_creator/creator.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/sonic_package_manager/service_creator/creator.py b/sonic_package_manager/service_creator/creator.py index 6299ed3747..5aa19c6c49 100644 --- a/sonic_package_manager/service_creator/creator.py +++ b/sonic_package_manager/service_creator/creator.py @@ -315,18 +315,16 @@ def generate_dump_script(self, package): log.info(f'generated {script_path}') def generate_shutdown_sequence(self, installed_packages, reboot_type): - shutdown_graph = dict() + shutdown_graph = defaultdict(set) for package in installed_packages: after = set(package.manifest['service'][f'{reboot_type}-shutdown']['after']) before = set(package.manifest['service'][f'{reboot_type}-shutdown']['before']) if not after and not before: continue name = package.manifest['service']['name'] - shutdown_graph.setdefault(name, set()) shutdown_graph[name].update(after) for service in before: - shutdown_graph.setdefault(service, set()) shutdown_graph[service].update({name}) log.debug(f'shutdown graph {pformat(shutdown_graph)}') From a1a97f958e744fdf882f942a096110d881711a36 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Tue, 30 Mar 2021 18:03:32 +0300 Subject: [PATCH 020/173] fix internal review comments Signed-off-by: Stepan Blyschak --- scripts/fast-reboot | 8 +- sonic_package_manager/manager.py | 34 ++++++--- .../service_creator/creator.py | 75 ++++++++++++++----- tests/sonic_package_manager/conftest.py | 11 +++ .../test_service_creator.py | 27 ++++--- 5 files changed, 114 insertions(+), 41 deletions(-) diff --git a/scripts/fast-reboot b/scripts/fast-reboot index d11b143d56..e67fded528 100755 --- a/scripts/fast-reboot +++ b/scripts/fast-reboot @@ -549,20 +549,20 @@ fi if [[ -f ${SHUTDOWN_ORDER_FILE} ]]; then - SERVICE_TO_STOP="$(cat ${SHUTDOWN_ORDER_FILE})" + SERVICES_TO_STOP="$(cat ${SHUTDOWN_ORDER_FILE})" else # TODO: to be removed once sonic-buildimage change is in if [[ "${REBOOT_TYPE}" == "fast-reboot" ]]; then - SERVICE_TO_STOP="nat radv bgp sflow lldp swss teamd syncd" + SERVICES_TO_STOP="nat radv bgp sflow lldp swss teamd syncd" elif [[ "${REBOOT_TYPE}" == "fastfast-reboot" || "${REBOOT_TYPE}" == "warm-reboot" ]]; then - SERVICE_TO_STOP="nat radv bgp sflow lldp teamd swss syncd" + SERVICES_TO_STOP="nat radv bgp sflow lldp teamd swss syncd" else error "Unexpected reboot type ${REBOOT_TYPE}" exit $EXIT_FAILURE fi fi -for service in ${SERVICE_TO_STOP}; do +for service in ${SERVICES_TO_STOP}; do debug "Stopping ${service} ..." # TODO: These exceptions for nat, sflow, lldp diff --git a/sonic_package_manager/manager.py b/sonic_package_manager/manager.py index 75390f30cd..660da49093 100644 --- a/sonic_package_manager/manager.py +++ b/sonic_package_manager/manager.py @@ -4,7 +4,14 @@ import os import pkgutil import tempfile -from typing import Any, Iterable, Callable, Dict, Optional +from typing import ( + Any, + Iterable, + List, + Callable, + Dict, + Optional, +) import docker import filelock @@ -345,12 +352,12 @@ def install_from_source(self, exit_stack.callback(rollback_wrapper(source.uninstall, package)) self.service_creator.create(package, - installed_packages.values(), + self.get_installed_packages().values(), state=feature_state, owner=default_owner) exit_stack.callback(rollback_wrapper(self.service_creator.remove, package, - installed_packages.values())) + self.get_installed_packages().values())) if not skip_cli_plugin_installation: self._install_cli_plugins(package) @@ -404,7 +411,7 @@ def uninstall(self, name: str, force=False): try: self._uninstall_cli_plugins(package) - self.service_creator.remove(package, installed_packages.values()) + self.service_creator.remove(package, self.get_installed_packages().values()) # Clean containers based on this image containers = self.docker.ps(filters={'ancestor': package.image_id}, all=True) @@ -515,11 +522,11 @@ def upgrade_from_source(self, old_package, 'start')) self.service_creator.remove(old_package, - installed_packages.values(), + self.get_installed_packages().values(), deregister_feature=False) exit_stack.callback(rollback_wrapper(self.service_creator.create, old_package, - installed_packages.values(), + self.get_installed_packages().values(), register_feature=False)) # This is no return point, after we start removing old Docker images @@ -533,7 +540,7 @@ def upgrade_from_source(self, self.docker.rmi(old_package.image_id, force=True) self.service_creator.create(new_package, - installed_packages.values(), + self.get_installed_packages().values(), register_feature=False) if self.feature_registry.is_feature_enabled(new_feature): @@ -793,10 +800,19 @@ def get_installed_packages(self) -> Dict[str, Package]: """ return { - entry.name: self.get_installed_package(entry.name) - for entry in self.database if entry.installed + entry.name: entry for entry in self.get_installed_packages_list() } + def get_installed_packages_list(self) -> List[Package]: + """ Returns a list of installed packages. + + Returns: + Installed packages dictionary. + """ + + return [self.get_installed_package(entry.name) + for entry in self.database if entry.installed] + def _migrate_package_database(self, old_package_database: PackageDatabase): """ Performs part of package migration process. For every package in old_package_database that is not listed in current diff --git a/sonic_package_manager/service_creator/creator.py b/sonic_package_manager/service_creator/creator.py index 5aa19c6c49..a7b95cbf69 100644 --- a/sonic_package_manager/service_creator/creator.py +++ b/sonic_package_manager/service_creator/creator.py @@ -94,7 +94,7 @@ def run_command(command: str): shell=True, executable='/bin/bash', stdout=subprocess.PIPE) - (out, _) = proc.communicate() + (_, _) = proc.communicate() if proc.returncode != 0: raise ServiceCreatorError(f'Failed to execute "{command}"') @@ -109,11 +109,23 @@ def __init__(self, feature_registry: FeatureRegistry, sonic_db): def create(self, package: Package, - all_packages: List[Package] = None, - register_feature=True, - state='enabled', - owner='local'): - all_packages = all_packages or [] + all_packages: List[Package], + register_feature: bool = True, + state: str = 'enabled', + owner: str = 'local'): + """ Register package as SONiC service. + + Args: + package: Package object to install. + all_packages: List of installed packages. + register_feature: Wether to register this package in FEATURE table. + state: Default feature state. + owner: Default feature owner. + + Returns: + None + """ + try: self.generate_container_mgmt(package) self.generate_service_mgmt(package) @@ -125,20 +137,31 @@ def create(self, self.set_initial_config(package) - self.post_operation_hook(all_packages) + self.post_operation_hook(all_packages + [package]) if register_feature: self.feature_registry.register(package.manifest, state, owner) except (Exception, KeyboardInterrupt): - self.remove(package, not register_feature) + self.remove(package, all_packages + [package], + deregister_feature=not register_feature) raise def remove(self, package: Package, - all_packages: List[Package] = None, - deregister_feature=True): - all_packages = all_packages or [] + all_packages: List[Package], + deregister_feature: bool = True): + """ Uninstall SONiC service provided by the package. + + Args: + package: Package object to uninstall. + all_packages: List of installed packages. + deregister_feature: Wether to deregister this package from FEATURE table. + + Returns: + None + """ + name = package.manifest['service']['name'] def remove_file(path): @@ -156,6 +179,8 @@ def remove_file(path): self.update_dependent_list_file(package, remove=True) + # remove package that is going to be uninstalled from installed list + all_packages.remove(package) self.post_operation_hook(all_packages) if deregister_feature: @@ -314,13 +339,27 @@ def generate_dump_script(self, package): render_template(scrip_template, script_path, render_ctx, executable=True) log.info(f'generated {script_path}') - def generate_shutdown_sequence(self, installed_packages, reboot_type): + def generate_shutdown_sequence(self, all_packages, reboot_type): shutdown_graph = defaultdict(set) - for package in installed_packages: - after = set(package.manifest['service'][f'{reboot_type}-shutdown']['after']) - before = set(package.manifest['service'][f'{reboot_type}-shutdown']['before']) + + def service_exists(service): + for package in all_packages: + if package.manifest['service']['name'] == service: + return True + log.info(f'Service {service} is not installed, it is skipped...') + return False + + def filter_not_available(services): + return set(filter(service_exists, services)) + + for package in all_packages: + service_props = package.manifest['service'] + after = filter_not_available(service_props[f'{reboot_type}-shutdown']['after']) + before = filter_not_available(service_props[f'{reboot_type}-shutdown']['before']) + if not after and not before: continue + name = package.manifest['service']['name'] shutdown_graph[name].update(after) @@ -332,7 +371,7 @@ def generate_shutdown_sequence(self, installed_packages, reboot_type): try: order = toposort_flatten(shutdown_graph) except CircularDependencyError as err: - raise ServiceCreatorError(f'Circular dependency found in {reboot_type} shutdown graph: {err}') + raise ServiceCreatorError(f'Circular dependency found in {reboot_type} error: {err}') log.debug(f'shutdown order {pformat(order)}') return order @@ -348,8 +387,8 @@ def generate_shutdown_sequence_files(self, installed_packages): def generate_service_reconciliation_file(self, package): name = package.manifest['service']['name'] - processes = [process['name'] for process in package.manifest['processes'] - if process['reconciles']] + all_processes = package.manifest['processes'] + processes = [process['name'] for process in all_processes if process['reconciles']] with open(os.path.join(ETC_SONIC_PATH, f'{name}_reconcile'), 'w') as file: file.write(' '.join(processes)) diff --git a/tests/sonic_package_manager/conftest.py b/tests/sonic_package_manager/conftest.py index fc8005212d..d7a9db3e57 100644 --- a/tests/sonic_package_manager/conftest.py +++ b/tests/sonic_package_manager/conftest.py @@ -93,6 +93,7 @@ def __init__(self): } ], ) + self.add('docker-syncd', 'latest', 'syncd', '1.0.0') self.add('docker-teamd', 'latest', 'teamd', '1.0.0', components={ 'libswsscommon': Version.parse('1.0.0'), @@ -223,6 +224,16 @@ def fake_db(fake_metadata_resolver): installed=True, built_in=True ) + add_package( + content, + fake_metadata_resolver, + 'docker-syncd', + 'latest', + description='SONiC syncd service', + default_reference='1.0.0', + installed=True, + built_in=True + ) add_package( content, fake_metadata_resolver, diff --git a/tests/sonic_package_manager/test_service_creator.py b/tests/sonic_package_manager/test_service_creator.py index 006b1af4ea..37c83ed16a 100644 --- a/tests/sonic_package_manager/test_service_creator.py +++ b/tests/sonic_package_manager/test_service_creator.py @@ -31,6 +31,13 @@ def manifest(): 'dependent-of': ['swss'], 'asic-service': False, 'host-service': True, + 'warm-shutdown': { + 'before': ['syncd'], + 'after': ['swss'], + }, + 'fast-shutdown': { + 'before': ['swss'], + }, }, 'container': { 'privileged': True, @@ -59,8 +66,8 @@ def test_service_creator(sonic_fs, manifest, package_manager, mock_feature_regis creator = ServiceCreator(mock_feature_registry, mock_sonic_db) entry = PackageEntry('test', 'azure/sonic-test') package = Package(entry, Metadata(manifest)) - installed_packages = package_manager.get_installed_packages().values() - creator.create(package, all_packages=installed_packages) + installed_packages = list(package_manager.get_installed_packages().values()) + [package] + creator.create(package, installed_packages) assert sonic_fs.exists(os.path.join(ETC_SONIC_PATH, 'swss_dependent')) assert sonic_fs.exists(os.path.join(DOCKER_CTL_SCRIPT_LOCATION, 'test.sh')) @@ -72,8 +79,8 @@ def read_file(name): with open(os.path.join(ETC_SONIC_PATH, name)) as file: return file.read() - assert read_file('warm-reboot_order') == 'swss teamd syncd' - assert read_file('fast-reboot_order') == 'teamd swss syncd' + assert read_file('warm-reboot_order') == 'swss teamd test syncd' + assert read_file('fast-reboot_order') == 'teamd test swss syncd' assert read_file('test_reconcile') == 'test-process test-process-3' @@ -81,13 +88,13 @@ def test_service_creator_with_timer_unit(sonic_fs, manifest, mock_feature_regist creator = ServiceCreator(mock_feature_registry, mock_sonic_db) entry = PackageEntry('test', 'azure/sonic-test') package = Package(entry, Metadata(manifest)) - creator.create(package) + creator.create(package, []) assert not sonic_fs.exists(os.path.join(SYSTEMD_LOCATION, 'test.timer')) manifest['service']['delayed'] = True package = Package(entry, Metadata(manifest)) - creator.create(package) + creator.create(package, []) assert sonic_fs.exists(os.path.join(SYSTEMD_LOCATION, 'test.timer')) @@ -96,13 +103,13 @@ def test_service_creator_with_debug_dump(sonic_fs, manifest, mock_feature_regist creator = ServiceCreator(mock_feature_registry, mock_sonic_db) entry = PackageEntry('test', 'azure/sonic-test') package = Package(entry, Metadata(manifest)) - creator.create(package) + creator.create(package, []) assert not sonic_fs.exists(os.path.join(DEBUG_DUMP_SCRIPT_LOCATION, 'test')) manifest['package']['debug-dump'] = '/some/command' package = Package(entry, Metadata(manifest)) - creator.create(package) + creator.create(package, []) assert sonic_fs.exists(os.path.join(DEBUG_DUMP_SCRIPT_LOCATION, 'test')) @@ -118,7 +125,7 @@ def test_service_creator_initial_config(sonic_fs, manifest, mock_feature_registr entry = PackageEntry('test', 'azure/sonic-test') package = Package(entry, Metadata(manifest)) - creator.create(package) + creator.create(package, []) assert not sonic_fs.exists(os.path.join(DEBUG_DUMP_SCRIPT_LOCATION, 'test')) @@ -132,7 +139,7 @@ def test_service_creator_initial_config(sonic_fs, manifest, mock_feature_registr } package = Package(entry, Metadata(manifest)) - creator.create(package) + creator.create(package, []) mock_table.set.assert_called_with('key_a', [('field_1', 'value_1'), ('field_2', 'original_value_2')]) From 7a6c8d145301668b3b4e0b11173dade618faca8d Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Tue, 30 Mar 2021 18:10:13 +0300 Subject: [PATCH 021/173] fix internal review comments Signed-off-by: Stepan Blyschak --- scripts/generate_shutdown_order.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/generate_shutdown_order.py b/scripts/generate_shutdown_order.py index a2427b3691..db9f48d676 100644 --- a/scripts/generate_shutdown_order.py +++ b/scripts/generate_shutdown_order.py @@ -6,7 +6,7 @@ def main(): manager = PackageManager.get_manager() - installed_packages = manager.get_installed_packages().values() + installed_packages = manager.get_installed_packages_list() print('installed packages {}'.format(installed_packages)) manager.service_creator.generate_shutdown_sequence_files(installed_packages) print('Done.') From c4b4225d1199bdfbf766b51e6310ed349f4f7efe Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Wed, 31 Mar 2021 14:57:08 +0300 Subject: [PATCH 022/173] use get_installed_packages_list() function Signed-off-by: Stepan Blyschak --- sonic_package_manager/manager.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/sonic_package_manager/manager.py b/sonic_package_manager/manager.py index 660da49093..53d632a59a 100644 --- a/sonic_package_manager/manager.py +++ b/sonic_package_manager/manager.py @@ -352,12 +352,12 @@ def install_from_source(self, exit_stack.callback(rollback_wrapper(source.uninstall, package)) self.service_creator.create(package, - self.get_installed_packages().values(), + self.get_installed_packages_list(), state=feature_state, owner=default_owner) exit_stack.callback(rollback_wrapper(self.service_creator.remove, package, - self.get_installed_packages().values())) + self.get_installed_packages_list())) if not skip_cli_plugin_installation: self._install_cli_plugins(package) @@ -411,7 +411,7 @@ def uninstall(self, name: str, force=False): try: self._uninstall_cli_plugins(package) - self.service_creator.remove(package, self.get_installed_packages().values()) + self.service_creator.remove(package, self.get_installed_packages_list()) # Clean containers based on this image containers = self.docker.ps(filters={'ancestor': package.image_id}, all=True) @@ -522,11 +522,11 @@ def upgrade_from_source(self, old_package, 'start')) self.service_creator.remove(old_package, - self.get_installed_packages().values(), + self.get_installed_packages_list(), deregister_feature=False) exit_stack.callback(rollback_wrapper(self.service_creator.create, old_package, - self.get_installed_packages().values(), + self.get_installed_packages_list(), register_feature=False)) # This is no return point, after we start removing old Docker images @@ -540,7 +540,7 @@ def upgrade_from_source(self, self.docker.rmi(old_package.image_id, force=True) self.service_creator.create(new_package, - self.get_installed_packages().values(), + self.get_installed_packages_list(), register_feature=False) if self.feature_registry.is_feature_enabled(new_feature): From f32d038de8ef6ed49a54b6d5cc09549248bfe7f5 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Wed, 31 Mar 2021 18:11:00 +0300 Subject: [PATCH 023/173] dont fail uninstall when no package in installed list Signed-off-by: Stepan Blyschak --- sonic_package_manager/service_creator/creator.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sonic_package_manager/service_creator/creator.py b/sonic_package_manager/service_creator/creator.py index a7b95cbf69..6622720168 100644 --- a/sonic_package_manager/service_creator/creator.py +++ b/sonic_package_manager/service_creator/creator.py @@ -143,7 +143,7 @@ def create(self, self.feature_registry.register(package.manifest, state, owner) except (Exception, KeyboardInterrupt): - self.remove(package, all_packages + [package], + self.remove(package, all_packages, deregister_feature=not register_feature) raise @@ -179,8 +179,8 @@ def remove_file(path): self.update_dependent_list_file(package, remove=True) - # remove package that is going to be uninstalled from installed list - all_packages.remove(package) + # make sure package that is going to be uninstalled is not in installed list. + with contextlib.suppress(ValueError): all_packages.remove(package) self.post_operation_hook(all_packages) if deregister_feature: From 83f2af68a1d4dec87649b7918c4ed7ec2206fd73 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Thu, 1 Apr 2021 12:40:53 +0300 Subject: [PATCH 024/173] drop support for monit as monit is getting deprecated Signed-off-by: Stepan Blyschak --- sonic-utilities-data/templates/monit.conf.j2 | 18 ------------------ sonic_package_manager/constraint.py | 1 - sonic_package_manager/manifest.py | 2 -- .../service_creator/creator.py | 16 ---------------- tests/sonic_package_manager/conftest.py | 2 -- .../test_service_creator.py | 7 +------ 6 files changed, 1 insertion(+), 45 deletions(-) delete mode 100644 sonic-utilities-data/templates/monit.conf.j2 diff --git a/sonic-utilities-data/templates/monit.conf.j2 b/sonic-utilities-data/templates/monit.conf.j2 deleted file mode 100644 index f51efb9bee..0000000000 --- a/sonic-utilities-data/templates/monit.conf.j2 +++ /dev/null @@ -1,18 +0,0 @@ -############################################################################### -## -## =============== Managed by SONiC Package Manager. DO NOT EDIT! =============== -## auto-generated from {{ source }} by sonic-package-manager -## -## Monit configuration for {{ feature }} service -## process list: -{%- for process in processes %} -{%- if process.critical %} -## {{ process.name }} -{%- endif %} -{%- endfor %} -############################################################################### -{%- for process in processes %} -check program {{ feature }}|{{ process.name }} with path "/usr/bin/process_checker {{ feature }} {{ process.command }}" - if status != 0 for 5 times within 5 cycles then alert - -{% endfor %} diff --git a/sonic_package_manager/constraint.py b/sonic_package_manager/constraint.py index 09f0fbc0fe..7dde13377f 100644 --- a/sonic_package_manager/constraint.py +++ b/sonic_package_manager/constraint.py @@ -137,4 +137,3 @@ def parse(constraint: Union[str, Dict]) -> 'PackageConstraint': return PackageConstraint.from_dict(constraint) else: raise ValueError('Input argument should be either str or dict') - diff --git a/sonic_package_manager/manifest.py b/sonic_package_manager/manifest.py index 5a357d668d..371ed145ba 100644 --- a/sonic_package_manager/manifest.py +++ b/sonic_package_manager/manifest.py @@ -184,9 +184,7 @@ def unmarshal(self, value): ManifestArray('tmpfs', DefaultMarshaller(str)), ]), ManifestArray('processes', ManifestRoot('processes', [ - ManifestField('critical', DefaultMarshaller(bool)), ManifestField('name', DefaultMarshaller(str)), - ManifestField('command', DefaultMarshaller(str)), ])), ManifestRoot('cli', [ ManifestField('mandatory', DefaultMarshaller(bool), False), diff --git a/sonic_package_manager/service_creator/creator.py b/sonic_package_manager/service_creator/creator.py index f62d0a3074..53fa51075d 100644 --- a/sonic_package_manager/service_creator/creator.py +++ b/sonic_package_manager/service_creator/creator.py @@ -25,9 +25,6 @@ DOCKER_CTL_SCRIPT_TEMPLATE = 'docker_image_ctl.j2' DOCKER_CTL_SCRIPT_LOCATION = '/usr/bin' -MONIT_CONF_TEMPLATE = 'monit.conf.j2' -MONIT_CONF_LOCATION = '/etc/monit/conf.d/' - DEBUG_DUMP_SCRIPT_TEMPLATE = 'dump.sh.j2' DEBUG_DUMP_SCRIPT_LOCATION = '/usr/local/bin/debug-dump/' @@ -116,7 +113,6 @@ def create(self, self.generate_service_mgmt(package) self.update_dependent_list_file(package) self.generate_systemd_service(package) - self.generate_monit_conf(package) self.generate_dump_script(package) self.set_initial_config(package) @@ -138,7 +134,6 @@ def remove_file(path): os.remove(path) log.info(f'removed {path}') - remove_file(os.path.join(MONIT_CONF_LOCATION, f'monit_{name}')) remove_file(os.path.join(SYSTEMD_LOCATION, f'{name}.service')) remove_file(os.path.join(SYSTEMD_LOCATION, f'{name}@.service')) remove_file(os.path.join(SERVICE_MGMT_SCRIPT_LOCATION, f'{name}.sh')) @@ -155,7 +150,6 @@ def remove_file(path): def post_operation_hook(self): if not in_chroot(): run_command('systemctl daemon-reload') - run_command('systemctl reload monit') def generate_container_mgmt(self, package: Package): image_id = package.image_id @@ -243,16 +237,6 @@ def generate_systemd_service(self, package: Package): render_template(template, output_file, template_vars) log.info(f'generated {output_file}') - def generate_monit_conf(self, package: Package): - name = package.manifest['service']['name'] - processes = package.manifest['processes'] - output_filename = os.path.join(MONIT_CONF_LOCATION, f'monit_{name}') - render_template(get_tmpl_path(MONIT_CONF_TEMPLATE), output_filename, - {'source': get_tmpl_path(MONIT_CONF_TEMPLATE), - 'feature': name, - 'processes': processes}) - log.info(f'generated {output_filename}') - def update_dependent_list_file(self, package: Package, remove=False): name = package.manifest['service']['name'] dependent_of = package.manifest['service']['dependent-of'] diff --git a/tests/sonic_package_manager/conftest.py b/tests/sonic_package_manager/conftest.py index bf787bdab4..d29f449684 100644 --- a/tests/sonic_package_manager/conftest.py +++ b/tests/sonic_package_manager/conftest.py @@ -337,12 +337,10 @@ def sonic_fs(fs): fs.create_dir(SYSTEMD_LOCATION) fs.create_dir(DOCKER_CTL_SCRIPT_LOCATION) fs.create_dir(SERVICE_MGMT_SCRIPT_LOCATION) - fs.create_dir(MONIT_CONF_LOCATION) fs.create_file(os.path.join(TEMPLATES_PATH, SERVICE_FILE_TEMPLATE)) fs.create_file(os.path.join(TEMPLATES_PATH, TIMER_UNIT_TEMPLATE)) fs.create_file(os.path.join(TEMPLATES_PATH, SERVICE_MGMT_SCRIPT_TEMPLATE)) fs.create_file(os.path.join(TEMPLATES_PATH, DOCKER_CTL_SCRIPT_TEMPLATE)) - fs.create_file(os.path.join(TEMPLATES_PATH, MONIT_CONF_TEMPLATE)) fs.create_file(os.path.join(TEMPLATES_PATH, DEBUG_DUMP_SCRIPT_TEMPLATE)) yield fs diff --git a/tests/sonic_package_manager/test_service_creator.py b/tests/sonic_package_manager/test_service_creator.py index e69b4447f8..f6ef317d8c 100644 --- a/tests/sonic_package_manager/test_service_creator.py +++ b/tests/sonic_package_manager/test_service_creator.py @@ -8,11 +8,7 @@ from sonic_package_manager.manifest import Manifest from sonic_package_manager.metadata import Metadata from sonic_package_manager.package import Package -from sonic_package_manager.service_creator.creator import ( - ServiceCreator, - ETC_SONIC_PATH, DOCKER_CTL_SCRIPT_LOCATION, - SERVICE_MGMT_SCRIPT_LOCATION, SYSTEMD_LOCATION, MONIT_CONF_LOCATION, DEBUG_DUMP_SCRIPT_LOCATION -) +from sonic_package_manager.service_creator.creator import * from sonic_package_manager.service_creator.feature import FeatureRegistry @@ -51,7 +47,6 @@ def test_service_creator(sonic_fs, manifest, mock_feature_registry, mock_sonic_d assert sonic_fs.exists(os.path.join(DOCKER_CTL_SCRIPT_LOCATION, 'test.sh')) assert sonic_fs.exists(os.path.join(SERVICE_MGMT_SCRIPT_LOCATION, 'test.sh')) assert sonic_fs.exists(os.path.join(SYSTEMD_LOCATION, 'test.service')) - assert sonic_fs.exists(os.path.join(MONIT_CONF_LOCATION, 'monit_test')) def test_service_creator_with_timer_unit(sonic_fs, manifest, mock_feature_registry, mock_sonic_db): From d31497ab6dfb76bba11564e700e5133ae5ce388f Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Thu, 1 Apr 2021 16:32:53 +0300 Subject: [PATCH 025/173] fix manifest representation Signed-off-by: Stepan Blyschak --- sonic_package_manager/constraint.py | 37 +++++++++++++++++--- sonic_package_manager/manifest.py | 2 ++ tests/sonic_package_manager/test_manifest.py | 8 ++++- 3 files changed, 41 insertions(+), 6 deletions(-) diff --git a/sonic_package_manager/constraint.py b/sonic_package_manager/constraint.py index 7dde13377f..8b044f3ec9 100644 --- a/sonic_package_manager/constraint.py +++ b/sonic_package_manager/constraint.py @@ -49,6 +49,18 @@ def parse(constraints: Dict) -> 'ComponentConstraints': for component, version in constraints.items()} return ComponentConstraints(components) + def deparse(self) -> Dict[str, str]: + """ Returns the manifest representation of components constraints. + + Returns: + Dictionary of string keys and string values. + + """ + + return { + component: str(version) for component, version in self.components + } + @dataclass class PackageConstraint: @@ -56,10 +68,12 @@ class PackageConstraint: name: str constraint: VersionConstraint - components: Dict[str, VersionConstraint] = field(default_factory=dict) + _components: ComponentConstraints = ComponentConstraints({}) + + def __str__(self): return f'{self.name}{self.constraint}' - def __str__(self): - return f'{self.name}{self.constraint}' + @property + def components(self): return self._components.components @staticmethod def from_string(constraint_expression: str) -> 'PackageConstraint': @@ -115,8 +129,7 @@ def from_dict(constraint_dict: Dict) -> 'PackageConstraint': name = constraint_dict['name'] version = VersionConstraint.parse(constraint_dict.get('version') or '*') - components = {component: VersionConstraint.parse(version) - for component, version in constraint_dict.get('components', {}).items()} + components = ComponentConstraints.parse(constraint_dict.get('components', {})) return PackageConstraint(name, version, components) @staticmethod @@ -137,3 +150,17 @@ def parse(constraint: Union[str, Dict]) -> 'PackageConstraint': return PackageConstraint.from_dict(constraint) else: raise ValueError('Input argument should be either str or dict') + + def deparse(self) -> Dict: + """ Returns the manifest representation of package constraint. + + Returns: + Dictionary in manifest representation. + + """ + + return { + 'name': self.name, + 'version': str(self.constraint), + 'components': self._components.deparse(), + } diff --git a/sonic_package_manager/manifest.py b/sonic_package_manager/manifest.py index 371ed145ba..451b28df16 100644 --- a/sonic_package_manager/manifest.py +++ b/sonic_package_manager/manifest.py @@ -54,6 +54,8 @@ def marshal(self, value): def unmarshal(self, value): try: + if hasattr(value, 'deparse'): + return value.deparse() return str(value) except Exception as err: raise ManifestError(f'Failed to unmarshal {value}: {err}') diff --git a/tests/sonic_package_manager/test_manifest.py b/tests/sonic_package_manager/test_manifest.py index cf3c02b6d1..efdcc558ab 100644 --- a/tests/sonic_package_manager/test_manifest.py +++ b/tests/sonic_package_manager/test_manifest.py @@ -59,7 +59,13 @@ def test_manifest_v1_mounts_invalid(): def test_manifest_v1_unmarshal(): manifest_json_input = {'package': {'name': 'test', 'version': '1.0.0', - 'depends': ['swss>1.0.0']}, + 'depends': [ + { + 'name': 'swss', + 'version': '>1.0.0', + 'components': {}, + } + ]}, 'service': {'name': 'test'}} manifest = Manifest.marshal(manifest_json_input) manifest_json = manifest.unmarshal() From c2eacdff5265fe7595514e79354d3030b553d7b9 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Thu, 1 Apr 2021 16:47:56 +0300 Subject: [PATCH 026/173] fix LGTM warning Signed-off-by: Stepan Blyschak --- sonic_package_manager/manifest.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/sonic_package_manager/manifest.py b/sonic_package_manager/manifest.py index 451b28df16..1bd5449d3a 100644 --- a/sonic_package_manager/manifest.py +++ b/sonic_package_manager/manifest.py @@ -5,11 +5,10 @@ from sonic_package_manager.constraint import ( ComponentConstraints, - PackageConstraint, - VersionConstraint + PackageConstraint ) from sonic_package_manager.errors import ManifestError -from sonic_package_manager.version import Version, VersionRange +from sonic_package_manager.version import Version class ManifestSchema: From f5ba2462be37cfcd8b98af2dba910d9af75adec3 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Fri, 9 Apr 2021 13:40:00 +0300 Subject: [PATCH 027/173] fix review comments Signed-off-by: Stepan Blyschak --- doc/Command-Reference.md | 2 +- sonic-utilities-data/bash_completion.d/spm | 9 +-------- 2 files changed, 2 insertions(+), 9 deletions(-) mode change 100644 => 120000 sonic-utilities-data/bash_completion.d/spm diff --git a/doc/Command-Reference.md b/doc/Command-Reference.md index 23095a3fc5..a05605c60c 100644 --- a/doc/Command-Reference.md +++ b/doc/Command-Reference.md @@ -7963,7 +7963,7 @@ Go Back To [Beginning of the document](#) or [Beginning of this section](#waterm ## Software Installation and Management SONiC image can be installed in one of two methods: -1. From within a running SONiC iamge using the `sonic-installer` utility +1. From within a running SONiC image using the `sonic-installer` utility 2. From the vendor's bootloader (E.g., ONIE, Aboot, etc.) SONiC feature Docker images (aka "SONiC packages") available to be installed with *sonic-package-manager* utility. diff --git a/sonic-utilities-data/bash_completion.d/spm b/sonic-utilities-data/bash_completion.d/spm deleted file mode 100644 index 8931dc389c..0000000000 --- a/sonic-utilities-data/bash_completion.d/spm +++ /dev/null @@ -1,8 +0,0 @@ -_sonic_package_manager_completion() { - COMPREPLY=( $( env COMP_WORDS="${COMP_WORDS[*]}" \ - COMP_CWORD=$COMP_CWORD \ - _SONIC_PACKAGE_MANAGER_COMPLETE=complete $1 ) ) - return 0 -} - -complete -F _sonic_package_manager_completion -o default spm; diff --git a/sonic-utilities-data/bash_completion.d/spm b/sonic-utilities-data/bash_completion.d/spm new file mode 120000 index 0000000000..3fff069223 --- /dev/null +++ b/sonic-utilities-data/bash_completion.d/spm @@ -0,0 +1 @@ +sonic-package-manager \ No newline at end of file From 2dee5c686b2a8aeae701276e640a33cee99568e4 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Wed, 14 Apr 2021 17:27:08 +0300 Subject: [PATCH 028/173] fix review comments Signed-off-by: Stepan Blyschak --- sonic_package_manager/main.py | 24 +++++++++++---------- sonic_package_manager/manager.py | 16 +++++++------- tests/sonic_package_manager/test_manager.py | 4 ++-- 3 files changed, 23 insertions(+), 21 deletions(-) diff --git a/sonic_package_manager/main.py b/sonic_package_manager/main.py index 713113da61..025d9d9ff3 100644 --- a/sonic_package_manager/main.py +++ b/sonic_package_manager/main.py @@ -109,11 +109,11 @@ def handle_parse_result(self, ctx, opts, args): PACKAGE_COMMON_INSTALL_OPTIONS = [ - click.option('--skip-cli-plugin-installation', + click.option('--skip-host-plugins', is_flag=True, - help='Do not install CLI plugins provided by the package ' - 'on the host OS. NOTE: In case when package /cli/mandatory ' - 'field is set to True this option will fail the installation.'), + help='Do not install host OS plugins provided by the package (CLI, etc). ' + 'NOTE: In case when package host OS plugins are set as mandatory in ' + 'package manifest this option will fail the installation.'), ] @@ -286,7 +286,9 @@ def changelog(ctx, @click.pass_context @root_privileges_required def add(ctx, name, repository, default_reference, description): - """ Add a new repository to database. """ + """ Add a new repository to database. + Repository in Docker Registry V2. + """ manager: PackageManager = ctx.obj @@ -338,8 +340,8 @@ def install(ctx, yes, enable, default_owner, - skip_cli_plugin_installation): - """ Install package """ + skip_host_plugins): + """ Install package using [PACKAGE_EXPR] in format "==" """ manager: PackageManager = ctx.obj @@ -353,7 +355,7 @@ def install(ctx, 'force': force, 'enable': enable, 'default_owner': default_owner, - 'skip_cli_plugin_installation': skip_cli_plugin_installation, + 'skip_host_plugins': skip_host_plugins, } try: @@ -379,8 +381,8 @@ def upgrade(ctx, from_tarball, force, yes, - skip_cli_plugin_installation): - """ Upgrade package """ + skip_host_plugins): + """ Upgrade package using [PACKAGE_EXPR] in format "==" """ manager: PackageManager = ctx.obj @@ -392,7 +394,7 @@ def upgrade(ctx, upgrade_opts = { 'force': force, - 'skip_cli_plugin_installation': skip_cli_plugin_installation, + 'skip_host_plugins': skip_host_plugins, } try: diff --git a/sonic_package_manager/manager.py b/sonic_package_manager/manager.py index f1643a1846..f0ff63add7 100644 --- a/sonic_package_manager/manager.py +++ b/sonic_package_manager/manager.py @@ -302,7 +302,7 @@ def install_from_source(self, force=False, enable=False, default_owner='local', - skip_cli_plugin_installation=False): + skip_host_plugins=False): """ Install SONiC Package from source represented by PackageSource. This method contains the logic of package installation. @@ -311,7 +311,7 @@ def install_from_source(self, force: Force the installation. enable: If True the installed feature package will be enabled. default_owner: Owner of the installed package. - skip_cli_plugin_installation: Skip CLI plugin installation. + skip_host_plugins: Skip CLI plugin installation. Raises: PackageManagerError """ @@ -330,7 +330,7 @@ def install_from_source(self, with failure_ignore(force): validate_package_base_os_constraints(package, self.version_info) validate_package_tree(installed_packages) - validate_package_cli_can_be_skipped(package, skip_cli_plugin_installation) + validate_package_cli_can_be_skipped(package, skip_host_plugins) # After all checks are passed we proceed to actual installation @@ -347,7 +347,7 @@ def install_from_source(self, self.service_creator.create(package, state=feature_state, owner=default_owner) exit_stack.callback(rollback_wrapper(self.service_creator.remove, package)) - if not skip_cli_plugin_installation: + if not skip_host_plugins: self._install_cli_plugins(package) exit_stack.callback(rollback_wrapper(self._uninstall_cli_plugins, package)) @@ -442,7 +442,7 @@ def upgrade(self, def upgrade_from_source(self, source: PackageSource, force=False, - skip_cli_plugin_installation=False): + skip_host_plugins=False): """ Upgrade SONiC Package to a version the package reference expression specifies. Can force the upgrade if force parameter is True. Force can allow a package downgrade. @@ -450,7 +450,7 @@ def upgrade_from_source(self, Args: source: SONiC Package source force: Force the upgrade. - skip_cli_plugin_installation: Skip CLI plugin installation. + skip_host_plugins: Skip host OS plugins installation. Raises: PackageManagerError """ @@ -492,7 +492,7 @@ def upgrade_from_source(self, with failure_ignore(force): validate_package_base_os_constraints(new_package, self.version_info) validate_package_tree(installed_packages) - validate_package_cli_can_be_skipped(new_package, skip_cli_plugin_installation) + validate_package_cli_can_be_skipped(new_package, skip_host_plugins) # After all checks are passed we proceed to actual upgrade @@ -528,7 +528,7 @@ def upgrade_from_source(self, if self.feature_registry.is_feature_enabled(new_feature): self._systemctl_action(new_package, 'start') - if not skip_cli_plugin_installation: + if not skip_host_plugins: self._install_cli_plugins(new_package) exit_stack.pop_all() diff --git a/tests/sonic_package_manager/test_manager.py b/tests/sonic_package_manager/test_manager.py index 256832118e..901f3a89fe 100644 --- a/tests/sonic_package_manager/test_manager.py +++ b/tests/sonic_package_manager/test_manager.py @@ -164,7 +164,7 @@ def test_installation_cli_plugin_skipped(package_manager, fake_metadata_resolver manifest = fake_metadata_resolver.metadata_store['Azure/docker-test']['1.6.0']['manifest'] manifest['cli']= {'show': '/cli/plugin.py'} package_manager._install_cli_plugins = Mock() - package_manager.install('test-package', skip_cli_plugin_installation=True) + package_manager.install('test-package', skip_host_plugins=True) package_manager._install_cli_plugins.assert_not_called() @@ -174,7 +174,7 @@ def test_installation_cli_plugin_is_mandatory_but_skipped(package_manager, fake_ with pytest.raises(PackageManagerError, match='CLI is mandatory for package test-package but ' 'it was requested to be not installed'): - package_manager.install('test-package', skip_cli_plugin_installation=True) + package_manager.install('test-package', skip_host_plugins=True) def test_installation(package_manager, mock_docker_api, anything): From 08c9c6d4c2cc5554bbd583d9128c607b760653ec Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Thu, 15 Apr 2021 12:25:28 +0300 Subject: [PATCH 029/173] [sonic_package_manager] add reset command Signed-off-by: Stepan Blyschak --- doc/Command-Reference.md | 137 +++++++++++--------- sonic_package_manager/main.py | 27 +++- sonic_package_manager/manager.py | 39 +++++- sonic_package_manager/source.py | 16 +-- tests/sonic_package_manager/conftest.py | 2 +- tests/sonic_package_manager/test_manager.py | 22 ++-- 6 files changed, 150 insertions(+), 93 deletions(-) diff --git a/doc/Command-Reference.md b/doc/Command-Reference.md index a05605c60c..7c7c58e8a5 100644 --- a/doc/Command-Reference.md +++ b/doc/Command-Reference.md @@ -8010,14 +8010,14 @@ This command will add a new entry in the package database. The package has to be - Usage: ``` - Usage: sonic-package-manager repository add [OPTIONS] NAME REPOSITORY +Usage: sonic-package-manager repository add [OPTIONS] NAME REPOSITORY - Add a new repository to database. + Add a new repository to database. Repository in Docker Registry V2. - Options: - --default-reference TEXT Default installation reference. - --description TEXT Optional package entry description. - --help Show this message and exit. +Options: + --default-reference TEXT Default installation reference + --description TEXT Optional package entry description + --help Show this message and exit. ``` - Example: ``` @@ -8049,41 +8049,34 @@ This command pulls and installs package on SONiC host. *NOTE*: this command requ - Usage: ``` - Usage: sonic-package-manager install [OPTIONS] [PACKAGE_EXPR] - - Install package - - Options: - --enable Set the default state of the feature to - enabled and enable feature right after - installation. NOTE: user needs to execute - "config save -y" to make this setting - persistent - - --default-owner [local|kube] Default owner configuration setting for a - feature - - --from-repository TEXT Fetch package directly from image registry - repository NOTE: This argument is mutually - exclusive with arguments: [from_tarball, - package_expr]. +Usage: sonic-package-manager install [OPTIONS] [PACKAGE_EXPR] - --from-tarball FILE Fetch package from saved image tarball - NOTE: This argument is mutually exclusive - with arguments: [from_repository, - package_expr]. + Install package using [PACKAGE_EXPR] in format "==" - -f, --force Force operation by ignoring failures - -y, --yes Automatically answer yes on prompts - -v, --verbosity LVL Either CRITICAL, ERROR, WARNING, INFO or - DEBUG - - --skip-cli-plugin-installation Do not install CLI plugins provided by the - package on the host OS. NOTE: In case when - package /cli/mandatory field is set to True - this option will fail the installation. - - --help Show this message and exit. +Options: + --enable Set the default state of the feature to + enabled and enable feature right after + installation. NOTE: user needs to execute + "config save -y" to make this setting + persistent + --default-owner [local|kube] Default owner configuration setting for a + feature + --from-repository TEXT Fetch package directly from image registry + repository NOTE: This argument is mutually + exclusive with arguments: [package_expr, + from_tarball]. + --from-tarball FILE Fetch package from saved image tarball NOTE: + This argument is mutually exclusive with + arguments: [package_expr, from_repository]. + -f, --force Force operation by ignoring failures + -y, --yes Automatically answer yes on prompts + -v, --verbosity LVL Either CRITICAL, ERROR, WARNING, INFO or DEBUG + --skip-host-plugins Do not install host OS plugins provided by the + package (CLI, etc). NOTE: In case when package + host OS plugins are set as mandatory in + package manifest this option will fail the + installation. + --help Show this message and exit. ``` - Example: ``` @@ -8124,32 +8117,25 @@ This command upgrades package on SONiC host to a newer version. The procedure of - Usage: ``` - Usage: sonic-package-manager upgrade [OPTIONS] [PACKAGE_EXPR] - - Upgrade package - - Options: - --from-repository TEXT Fetch package directly from image registry - repository NOTE: This argument is mutually - exclusive with arguments: [package_expr, - from_tarball]. - - --from-tarball FILE Fetch package from saved image tarball - NOTE: This argument is mutually exclusive - with arguments: [package_expr, - from_repository]. +Usage: sonic-package-manager upgrade [OPTIONS] [PACKAGE_EXPR] - -f, --force Force operation by ignoring failures - -y, --yes Automatically answer yes on prompts - -v, --verbosity LVL Either CRITICAL, ERROR, WARNING, INFO or - DEBUG + Upgrade package using [PACKAGE_EXPR] in format "==" - --skip-cli-plugin-installation Do not install CLI plugins provided by the - package on the host OS. NOTE: In case when - package /cli/mandatory field is set to True - this option will fail the installation. - - --help Show this message and exit. +Options: + --from-repository TEXT Fetch package directly from image registry + repository NOTE: This argument is mutually exclusive + with arguments: [package_expr, from_tarball]. + --from-tarball FILE Fetch package from saved image tarball NOTE: This + argument is mutually exclusive with arguments: + [from_repository, package_expr]. + -f, --force Force operation by ignoring failures + -y, --yes Automatically answer yes on prompts + -v, --verbosity LVL Either CRITICAL, ERROR, WARNING, INFO or DEBUG + --skip-host-plugins Do not install host OS plugins provided by the + package (CLI, etc). NOTE: In case when package host + OS plugins are set as mandatory in package manifest + this option will fail the installation. + --help Show this message and exit. ``` - Example: ``` @@ -8162,6 +8148,31 @@ This command upgrades package on SONiC host to a newer version. The procedure of admin@sonic:~$ sudo sonic-package-manager upgrade --from-tarball sonic-docker-image.gz ``` +**sonic-package-manager reset** + +This comamnd resets the package by reinstalling it to its default version. + +- Usage: + ``` +Usage: sonic-package-manager reset [OPTIONS] NAME + + Reset package to the default version + +Options: + -f, --force Force operation by ignoring failures + -y, --yes Automatically answer yes on prompts + -v, --verbosity LVL Either CRITICAL, ERROR, WARNING, INFO or DEBUG + --skip-host-plugins Do not install host OS plugins provided by the package + (CLI, etc). NOTE: In case when package host OS plugins + are set as mandatory in package manifest this option + will fail the installation. + --help Show this message and exit. + ``` +- Example: + ``` + admin@sonic:~$ sudo sonic-package-manager reset dhcp-relay + ``` + **sonic-package-manager show package versions** This command will access repository for corresponding package and retrieve a list of available versions. diff --git a/sonic_package_manager/main.py b/sonic_package_manager/main.py index 025d9d9ff3..13a59e8edb 100644 --- a/sonic_package_manager/main.py +++ b/sonic_package_manager/main.py @@ -281,8 +281,8 @@ def changelog(ctx, @repository.command() @click.argument('name', type=str) @click.argument('repository', type=str) -@click.option('--default-reference', type=str, help='Default installation reference.') -@click.option('--description', type=str, help='Default installation reference.') +@click.option('--default-reference', type=str, help='Default installation reference') +@click.option('--description', type=str, help='Optional package entry description') @click.pass_context @root_privileges_required def add(ctx, name, repository, default_reference, description): @@ -408,6 +408,29 @@ def upgrade(ctx, exit_cli(f'Operation canceled by user', fg='red') +@cli.command() +@add_options(PACKAGE_COMMON_OPERATION_OPTIONS) +@add_options(PACKAGE_COMMON_INSTALL_OPTIONS) +@click.argument('name') +@click.pass_context +@root_privileges_required +def reset(ctx, name, force, yes, skip_host_plugins): + """ Reset package to the default version """ + + manager: PackageManager = ctx.obj + + if not yes and not force: + click.confirm(f'Package {name} is going to be reset to default version, ' + f'continue?', abort=True, show_default=True) + + try: + manager.reset(name, force, skip_host_plugins) + except Exception as err: + exit_cli(f'Failed to reset package {name}: {err}', fg='red') + except KeyboardInterrupt: + exit_cli(f'Operation canceled by user', fg='red') + + @cli.command() @add_options(PACKAGE_COMMON_OPERATION_OPTIONS) @click.argument('name') diff --git a/sonic_package_manager/manager.py b/sonic_package_manager/manager.py index f0ff63add7..4077f875eb 100644 --- a/sonic_package_manager/manager.py +++ b/sonic_package_manager/manager.py @@ -442,7 +442,8 @@ def upgrade(self, def upgrade_from_source(self, source: PackageSource, force=False, - skip_host_plugins=False): + skip_host_plugins=False, + allow_downgrade=False): """ Upgrade SONiC Package to a version the package reference expression specifies. Can force the upgrade if force parameter is True. Force can allow a package downgrade. @@ -451,6 +452,7 @@ def upgrade_from_source(self, source: SONiC Package source force: Force the upgrade. skip_host_plugins: Skip host OS plugins installation. + allow_downgrade: Flag to allow package downgrade. Raises: PackageManagerError """ @@ -478,11 +480,9 @@ def upgrade_from_source(self, # TODO: Not all packages might support downgrade. # We put a check here but we understand that for some packages - # the downgrade might be safe to do. In that case we might want to - # add another argument to this function: allow_downgrade: bool = False. - # Another way to do that might be a variable in manifest describing package - # downgrade ability or downgrade-able versions. - if new_version < old_version: + # the downgrade might be safe to do. There can be a variable in manifest + # describing package downgrade ability or downgrade-able versions. + if new_version < old_version and not allow_downgrade: raise PackageUpgradeError(f'Request to downgrade from {old_version} to {new_version}. ' f'Downgrade might be not supported by the package') @@ -543,6 +543,33 @@ def upgrade_from_source(self, self.database.update_package(new_package_entry) self.database.commit() + @under_lock + def reset(self, name: str, force: bool = False, skip_host_plugins: bool = False): + """ Reset package to defaults version + + Args: + name: SONiC Package name. + force: Force the installation. + skip_host_plugins: Skip host plugins installation. + Raises: + PackageManagerError + """ + + with failure_ignore(force): + if not self.is_installed(name): + raise PackageManagerError(f'{name} is not installed') + + package = self.get_installed_package(name) + default_reference = package.entry.default_reference + if default_reference is None: + raise PackageManagerError(f'package {name} has no default reference') + + package_ref = PackageReference(name, default_reference) + source = self.get_package_source(package_ref=package_ref) + self.upgrade_from_source(source, force=force, + allow_downgrade=True, + skip_host_plugins=skip_host_plugins) + @under_lock def migrate_packages(self, old_package_database: PackageDatabase, diff --git a/sonic_package_manager/source.py b/sonic_package_manager/source.py index 4720b5be9e..40d2408ba7 100644 --- a/sonic_package_manager/source.py +++ b/sonic_package_manager/source.py @@ -76,20 +76,20 @@ def get_package(self) -> Package: name = manifest['package']['name'] description = manifest['package']['description'] + # Will be resolved in install() method. + # When installing from tarball we don't know yet + # the repository for this package. repository = None if self.database.has_package(name): # inherit package database info - package = self.database.get_package(name) - repository = package.repository - description = description or package.description + package_entry = self.database.get_package(name) + else: + package_entry = PackageEntry(name, repository, + description=description) return Package( - PackageEntry( - name, - repository, - description, - ), + package_entry, metadata ) diff --git a/tests/sonic_package_manager/conftest.py b/tests/sonic_package_manager/conftest.py index d29f449684..cee997596c 100644 --- a/tests/sonic_package_manager/conftest.py +++ b/tests/sonic_package_manager/conftest.py @@ -29,7 +29,7 @@ def attrs(self): return {'RepoTags': [self.id]} def pull(repo, ref): - return Image(f'{repo}:latest') + return Image(f'{repo}:{ref}') def load(filename): return Image(filename) diff --git a/tests/sonic_package_manager/test_manager.py b/tests/sonic_package_manager/test_manager.py index 901f3a89fe..08b957475a 100644 --- a/tests/sonic_package_manager/test_manager.py +++ b/tests/sonic_package_manager/test_manager.py @@ -269,19 +269,6 @@ def test_installation_fault(package_manager, mock_docker_api, mock_service_creat mock_docker_api.rmi.assert_called_once() -def test_installation_package_with_description(package_manager, fake_metadata_resolver): - package_entry = package_manager.database.get_package('test-package') - description = package_entry.description - references = fake_metadata_resolver.metadata_store[package_entry.repository] - manifest = references[package_entry.default_reference]['manifest'] - new_description = description + ' changed description ' - manifest['package']['description'] = new_description - package_manager.install('test-package') - package_entry = package_manager.database.get_package('test-package') - description = package_entry.description - assert description == new_description - - def test_manager_installation_version_range(package_manager): with pytest.raises(PackageManagerError, match='Can only install specific version. ' @@ -298,6 +285,15 @@ def test_manager_upgrade(package_manager, sonic_fs): assert upgraded_package.entry.version == Version(2, 0, 0) +def test_manager_package_reset(package_manager, sonic_fs): + package_manager.install('test-package-6==1.5.0') + package_manager.upgrade('test-package-6==2.0.0') + + package_manager.reset('test-package-6') + upgraded_package = package_manager.get_installed_package('test-package-6') + assert upgraded_package.entry.version == Version(1, 5, 0) + + def test_manager_migration(package_manager, fake_db_for_migration): package_manager.install = Mock() package_manager.upgrade = Mock() From 8bd4f5e46fcfc6f2c8a21fcee0d286aba37de97f Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Fri, 16 Apr 2021 17:04:59 +0300 Subject: [PATCH 030/173] add yang Signed-off-by: Stepan Blyschak --- config/config_mgmt.py | 96 +++++++++++++++++++++---- sonic_package_manager/manager.py | 23 +++++- sonic_package_manager/metadata.py | 7 +- tests/sonic_package_manager/conftest.py | 9 ++- 4 files changed, 115 insertions(+), 20 deletions(-) diff --git a/config/config_mgmt.py b/config/config_mgmt.py index 194c8aefc9..7d2e3dc06b 100644 --- a/config/config_mgmt.py +++ b/config/config_mgmt.py @@ -3,6 +3,7 @@ Port Breakout. ''' try: + import os import re import syslog @@ -53,33 +54,37 @@ def __init__(self, source="configDB", debug=False, allowTablesWithoutYang=True): try: self.configdbJsonIn = None self.configdbJsonOut = None + self.source = source self.allowTablesWithoutYang = allowTablesWithoutYang # logging vars self.SYSLOG_IDENTIFIER = "ConfigMgmt" self.DEBUG = debug - self.sy = sonic_yang.SonicYang(YANG_DIR, debug=debug) - # load yang models - self.sy.loadYangModel() - # load jIn from config DB or from config DB json file. - if source.lower() == 'configdb': - self.readConfigDB() - # treat any other source as file input - else: - self.readConfigDBJson(source) - # this will crop config, xlate and load. - self.sy.loadData(self.configdbJsonIn) - - # Raise if tables without YANG models are not allowed but exist. - if not allowTablesWithoutYang and len(self.sy.tablesWithOutYang): - raise Exception('Config has tables without YANG models') + self.__init_sonic_yang() except Exception as e: self.sysLog(doPrint=True, logLevel=syslog.LOG_ERR, msg=str(e)) raise Exception('ConfigMgmt Class creation failed') return + + def __init_sonic_yang(self): + self.sy = sonic_yang.SonicYang(YANG_DIR, debug=self.DEBUG) + # load yang models + self.sy.loadYangModel() + # load jIn from config DB or from config DB json file. + if self.source.lower() == 'configdb': + self.readConfigDB() + # treat any other source as file input + else: + self.readConfigDBJson(self.source) + # this will crop config, xlate and load. + self.sy.loadData(self.configdbJsonIn) + + # Raise if tables without YANG models are not allowed but exist. + if not self.allowTablesWithoutYang and len(self.sy.tablesWithOutYang): + raise Exception('Config has tables without YANG models') def __del__(self): pass @@ -219,6 +224,67 @@ def writeConfigDB(self, jDiff): configdb.mod_config(FormatConverter.output_to_db(data)) return + + def add_module(self, yang_module_text, allow_if_exists=False): + """ + Validate and add new YANG module to the system. + + Parameters: + yang_module_text (str): YANG module string. + + Returns: + None + """ + + module_name = self.get_module_name(yang_module_text) + module_path = os.path.join(YANG_DIR, '{}.yang'.format(module_name)) + if os.path.exists(module_path): + raise Exception('{} already exists'.format(module_name)) + try: + with open(module_path, 'w') as module_file: + module_file.write(yang_module_text) + self.__init_sonic_yang() + except Exception: + os.remove(module_path) + raise + + def remove_module(self, module_name): + """ + Remove YANG module on the system and validate. + + Parameters: + module_name (str): YANG module name. + + Returns: + None + """ + + module_path = os.path.join(YANG_DIR, '{}.yang'.format(module_name)) + try: + with open(module_path, 'r') as module_file: + yang_module_text = module_file.read() + os.remove(module_path) + self.__init_sonic_yang() + except Exception: + self.add_module(yang_module_text) + raise + + @staticmethod + def get_module_name(yang_module_text): + """ + Read yangs module name from yang_module_text + + Parameters: + yang_module_text(str): YANG module string. + + Returns: + str: Module name + """ + + sy = sonic_yang.SonicYang(YANG_DIR) + module = sy.ctx.parse_module_mem(yang_module_text, ly.LYS_IN_YANG) + return module.name() + # End of Class ConfigMgmt diff --git a/sonic_package_manager/manager.py b/sonic_package_manager/manager.py index 4077f875eb..055f4b8d1a 100644 --- a/sonic_package_manager/manager.py +++ b/sonic_package_manager/manager.py @@ -8,6 +8,7 @@ import docker import filelock +from config import config_mgmt from sonic_py_common import device_info from sonic_package_manager import utils @@ -235,7 +236,8 @@ def __init__(self, metadata_resolver: MetadataResolver, service_creator: ServiceCreator, device_information: Any, - lock: filelock.FileLock): + lock: filelock.FileLock, + cfg_mgmt: config_mgmt.ConfigMgmt): """ Initialize PackageManager. """ self.lock = lock @@ -248,6 +250,7 @@ def __init__(self, self.is_multi_npu = device_information.is_multi_npu() self.num_npus = device_information.get_num_npus() self.version_info = device_information.get_sonic_version_info() + self.cfg_mgmt = cfg_mgmt @under_lock def add_repository(self, *args, **kwargs): @@ -347,6 +350,9 @@ def install_from_source(self, self.service_creator.create(package, state=feature_state, owner=default_owner) exit_stack.callback(rollback_wrapper(self.service_creator.remove, package)) + self._install_yang_module(package) + exit_stack.callback(rollback_wrapper(self._uninstall_yang_module, package)) + if not skip_host_plugins: self._install_cli_plugins(package) exit_stack.callback(rollback_wrapper(self._uninstall_cli_plugins, package)) @@ -399,6 +405,7 @@ def uninstall(self, name: str, force=False): try: self._uninstall_cli_plugins(package) + self._uninstall_yang_module(package) self.service_creator.remove(package) # Clean containers based on this image @@ -524,6 +531,7 @@ def upgrade_from_source(self, self.docker.rmi(old_package.image_id, force=True) self.service_creator.create(new_package, register_feature=False) + self._upgrade_yang_module(new_package) if self.feature_registry.is_feature_enabled(new_feature): self._systemctl_action(new_package, 'start') @@ -880,6 +888,16 @@ def _get_cli_plugin_path(cls, package: Package, command): raise PackageManagerError(f'Failed to get plugins path for {command} CLI') plugins_pkg_path = os.path.dirname(pkg_loader.path) return os.path.join(plugins_pkg_path, cls._get_cli_plugin_name(package)) + + def _install_yang_module(self, package: Package): + self.cfg_mgmt.add_module(package.metadata.yang_module_text) + + def _upgrade_yang_module(self, package: Package): + self.cfg_mgmt.add_module(package.metadata.yang_module_text, allow_if_exists=True) + + def _uninstall_yang_module(self, package: Package): + module_name = self.cfg_mgmt.get_module_name(package.metadata.yang_module_text) + self.cfg_mgmt.remove_module(module_name) def _install_cli_plugins(self, package: Package): for command in ('show', 'config', 'clear'): @@ -920,4 +938,5 @@ def get_manager() -> 'PackageManager': MetadataResolver(docker_api, registry_resolver), ServiceCreator(FeatureRegistry(SonicDB), SonicDB), device_info, - filelock.FileLock(PACKAGE_MANAGER_LOCK_FILE, timeout=0)) + filelock.FileLock(PACKAGE_MANAGER_LOCK_FILE, timeout=0), + config_mgmt.ConfigMgmt()) diff --git a/sonic_package_manager/metadata.py b/sonic_package_manager/metadata.py index 7f7c25ceaf..c8f902b3c7 100644 --- a/sonic_package_manager/metadata.py +++ b/sonic_package_manager/metadata.py @@ -4,7 +4,7 @@ import json import tarfile -from typing import Dict +from typing import Dict, Optional from sonic_package_manager.errors import MetadataError from sonic_package_manager.manifest import Manifest @@ -73,6 +73,7 @@ class Metadata: manifest: Manifest components: Dict[str, Version] = field(default_factory=dict) + yang_module_text: Optional[str] = None class MetadataResolver: @@ -181,5 +182,7 @@ def from_labels(cls, labels: Dict[str, str]) -> Metadata: components[component] = Version.parse(version) except ValueError as err: raise MetadataError(f'Failed to parse component version: {err}') + + yang_module_text = sonic_metadata.get('yang-module') - return Metadata(Manifest.marshal(manifest_dict), components) + return Metadata(Manifest.marshal(manifest_dict), components, yang_module_text) diff --git a/tests/sonic_package_manager/conftest.py b/tests/sonic_package_manager/conftest.py index cee997596c..088fde27f1 100644 --- a/tests/sonic_package_manager/conftest.py +++ b/tests/sonic_package_manager/conftest.py @@ -65,6 +65,11 @@ def mock_sonic_db(): yield Mock() +@pytest.fixture +def mock_config_mgmt(): + yield Mock() + + @pytest.fixture def fake_metadata_resolver(): class FakeMetadataResolver: @@ -355,6 +360,7 @@ def patch_pkgutil(): def package_manager(mock_docker_api, mock_registry_resolver, mock_service_creator, + mock_config_mgmt, fake_metadata_resolver, fake_db, fake_device_info): @@ -362,7 +368,8 @@ def package_manager(mock_docker_api, fake_db, fake_metadata_resolver, mock_service_creator, fake_device_info, - MagicMock()) + MagicMock(), + mock_config_mgmt) @pytest.fixture From 413b44f092d8c99bd797fcf27a72d4adfde7ef81 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Fri, 16 Apr 2021 17:09:06 +0300 Subject: [PATCH 031/173] [doc] fix formatting Signed-off-by: Stepan Blyschak --- doc/Command-Reference.md | 126 +++++++++++++++++++-------------------- 1 file changed, 63 insertions(+), 63 deletions(-) diff --git a/doc/Command-Reference.md b/doc/Command-Reference.md index 7c7c58e8a5..18c9e7d252 100644 --- a/doc/Command-Reference.md +++ b/doc/Command-Reference.md @@ -7988,7 +7988,7 @@ SONiC package status can be *Installed*, *Not installed* or *Built-In*. "Built-I Name Repository Description Version Status -------------- --------------------------- ---------------------------- --------- --------- database docker-database SONiC database package 1.0.0 Built-In - dhcp-relay docker-dhcp-relay SONiC dhcp-relay package 1.0.0 Installed + dhcp-relay azure/docker-dhcp-relay SONiC dhcp-relay package 1.0.0 Installed fpm-frr docker-fpm-frr SONiC fpm-frr package 1.0.0 Built-In lldp docker-lldp SONiC lldp package 1.0.0 Built-In macsec docker-macsec SONiC macsec package 1.0.0 Built-In @@ -8006,18 +8006,18 @@ SONiC package status can be *Installed*, *Not installed* or *Built-In*. "Built-I **sonic-package-manager repository add** -This command will add a new entry in the package database. The package has to be *Not Installed* in order to be removed from package database. *NOTE*: this command requires elevated (root) privileges to run. +This command will add a new entry in the package database. *NOTE*: this command requires elevated (root) privileges to run. - Usage: ``` -Usage: sonic-package-manager repository add [OPTIONS] NAME REPOSITORY + Usage: sonic-package-manager repository add [OPTIONS] NAME REPOSITORY - Add a new repository to database. Repository in Docker Registry V2. + Add a new repository to database. Repository in Docker Registry V2. -Options: - --default-reference TEXT Default installation reference - --description TEXT Optional package entry description - --help Show this message and exit. + Options: + --default-reference TEXT Default installation reference + --description TEXT Optional package entry description + --help Show this message and exit. ``` - Example: ``` @@ -8027,7 +8027,7 @@ Options: **sonic-package-manager repository remove** -This command will remove an entry from the package database. *NOTE*: this command requires elevated (root) privileges to run. +This command will remove an entry from the package database. The package has to be *Not Installed* in order to be removed from package database. *NOTE*: this command requires elevated (root) privileges to run. - Usage: ``` @@ -8049,34 +8049,34 @@ This command pulls and installs package on SONiC host. *NOTE*: this command requ - Usage: ``` -Usage: sonic-package-manager install [OPTIONS] [PACKAGE_EXPR] + Usage: sonic-package-manager install [OPTIONS] [PACKAGE_EXPR] - Install package using [PACKAGE_EXPR] in format "==" + Install package using [PACKAGE_EXPR] in format "==" -Options: - --enable Set the default state of the feature to - enabled and enable feature right after - installation. NOTE: user needs to execute - "config save -y" to make this setting - persistent - --default-owner [local|kube] Default owner configuration setting for a - feature - --from-repository TEXT Fetch package directly from image registry - repository NOTE: This argument is mutually - exclusive with arguments: [package_expr, - from_tarball]. - --from-tarball FILE Fetch package from saved image tarball NOTE: - This argument is mutually exclusive with - arguments: [package_expr, from_repository]. - -f, --force Force operation by ignoring failures - -y, --yes Automatically answer yes on prompts - -v, --verbosity LVL Either CRITICAL, ERROR, WARNING, INFO or DEBUG - --skip-host-plugins Do not install host OS plugins provided by the - package (CLI, etc). NOTE: In case when package - host OS plugins are set as mandatory in - package manifest this option will fail the - installation. - --help Show this message and exit. + Options: + --enable Set the default state of the feature to + enabled and enable feature right after + installation. NOTE: user needs to execute + "config save -y" to make this setting + persistent + --default-owner [local|kube] Default owner configuration setting for a + feature + --from-repository TEXT Fetch package directly from image registry + repository NOTE: This argument is mutually + exclusive with arguments: [package_expr, + from_tarball]. + --from-tarball FILE Fetch package from saved image tarball NOTE: + This argument is mutually exclusive with + arguments: [package_expr, from_repository]. + -f, --force Force operation by ignoring failures + -y, --yes Automatically answer yes on prompts + -v, --verbosity LVL Either CRITICAL, ERROR, WARNING, INFO or DEBUG + --skip-host-plugins Do not install host OS plugins provided by the + package (CLI, etc). NOTE: In case when package + host OS plugins are set as mandatory in + package manifest this option will fail the + installation. + --help Show this message and exit. ``` - Example: ``` @@ -8117,25 +8117,25 @@ This command upgrades package on SONiC host to a newer version. The procedure of - Usage: ``` -Usage: sonic-package-manager upgrade [OPTIONS] [PACKAGE_EXPR] + Usage: sonic-package-manager upgrade [OPTIONS] [PACKAGE_EXPR] - Upgrade package using [PACKAGE_EXPR] in format "==" + Upgrade package using [PACKAGE_EXPR] in format "==" -Options: - --from-repository TEXT Fetch package directly from image registry - repository NOTE: This argument is mutually exclusive - with arguments: [package_expr, from_tarball]. - --from-tarball FILE Fetch package from saved image tarball NOTE: This - argument is mutually exclusive with arguments: - [from_repository, package_expr]. - -f, --force Force operation by ignoring failures - -y, --yes Automatically answer yes on prompts - -v, --verbosity LVL Either CRITICAL, ERROR, WARNING, INFO or DEBUG - --skip-host-plugins Do not install host OS plugins provided by the - package (CLI, etc). NOTE: In case when package host - OS plugins are set as mandatory in package manifest - this option will fail the installation. - --help Show this message and exit. + Options: + --from-repository TEXT Fetch package directly from image registry + repository NOTE: This argument is mutually exclusive + with arguments: [package_expr, from_tarball]. + --from-tarball FILE Fetch package from saved image tarball NOTE: This + argument is mutually exclusive with arguments: + [from_repository, package_expr]. + -f, --force Force operation by ignoring failures + -y, --yes Automatically answer yes on prompts + -v, --verbosity LVL Either CRITICAL, ERROR, WARNING, INFO or DEBUG + --skip-host-plugins Do not install host OS plugins provided by the + package (CLI, etc). NOTE: In case when package host + OS plugins are set as mandatory in package manifest + this option will fail the installation. + --help Show this message and exit. ``` - Example: ``` @@ -8154,19 +8154,19 @@ This comamnd resets the package by reinstalling it to its default version. - Usage: ``` -Usage: sonic-package-manager reset [OPTIONS] NAME + Usage: sonic-package-manager reset [OPTIONS] NAME - Reset package to the default version + Reset package to the default version -Options: - -f, --force Force operation by ignoring failures - -y, --yes Automatically answer yes on prompts - -v, --verbosity LVL Either CRITICAL, ERROR, WARNING, INFO or DEBUG - --skip-host-plugins Do not install host OS plugins provided by the package - (CLI, etc). NOTE: In case when package host OS plugins - are set as mandatory in package manifest this option - will fail the installation. - --help Show this message and exit. + Options: + -f, --force Force operation by ignoring failures + -y, --yes Automatically answer yes on prompts + -v, --verbosity LVL Either CRITICAL, ERROR, WARNING, INFO or DEBUG + --skip-host-plugins Do not install host OS plugins provided by the package + (CLI, etc). NOTE: In case when package host OS plugins + are set as mandatory in package manifest this option + will fail the installation. + --help Show this message and exit. ``` - Example: ``` From f101f1f97c49d6ee4456a4ccbae950b9de85476f Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Fri, 16 Apr 2021 17:18:40 +0300 Subject: [PATCH 032/173] install takes a digest or tag as well Signed-off-by: Stepan Blyschak --- doc/Command-Reference.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/Command-Reference.md b/doc/Command-Reference.md index 18c9e7d252..b85a368132 100644 --- a/doc/Command-Reference.md +++ b/doc/Command-Reference.md @@ -8051,7 +8051,7 @@ This command pulls and installs package on SONiC host. *NOTE*: this command requ ``` Usage: sonic-package-manager install [OPTIONS] [PACKAGE_EXPR] - Install package using [PACKAGE_EXPR] in format "==" + Install package using [PACKAGE_EXPR] in format "==" or "@" Options: --enable Set the default state of the feature to From 4d03aa89f66618b8b5096e14c045b0c213a3c016 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Tue, 20 Apr 2021 17:09:24 +0300 Subject: [PATCH 033/173] resolve review comments Signed-off-by: Stepan Blyschak --- doc/Command-Reference.md | 71 +++----- sonic_package_manager/main.py | 46 +---- sonic_package_manager/manager.py | 183 ++++++++++---------- sonic_package_manager/source.py | 15 +- tests/sonic_package_manager/test_manager.py | 42 +++-- 5 files changed, 161 insertions(+), 196 deletions(-) diff --git a/doc/Command-Reference.md b/doc/Command-Reference.md index b85a368132..01ab77acce 100644 --- a/doc/Command-Reference.md +++ b/doc/Command-Reference.md @@ -7986,7 +7986,8 @@ SONiC package status can be *Installed*, *Not installed* or *Built-In*. "Built-I ``` admin@sonic:~$ sonic-package-manager list Name Repository Description Version Status - -------------- --------------------------- ---------------------------- --------- --------- + -------------- --------------------------- ---------------------------- --------- -------------- + cpu-report azure/cpu-report CPU report package N/A Not Installed database docker-database SONiC database package 1.0.0 Built-In dhcp-relay azure/docker-dhcp-relay SONiC dhcp-relay package 1.0.0 Installed fpm-frr docker-fpm-frr SONiC fpm-frr package 1.0.0 Built-In @@ -8045,13 +8046,15 @@ This command will remove an entry from the package database. The package has to **sonic-package-manager install** -This command pulls and installs package on SONiC host. *NOTE*: this command requires elevated (root) privileges to run. +This command pulls and installs or upgrades (if already installed) package on SONiC host. *NOTE*: this command requires elevated (root) privileges to run. +The procedure of upgrading a package will restart the corresponding service. - Usage: ``` Usage: sonic-package-manager install [OPTIONS] [PACKAGE_EXPR] - Install package using [PACKAGE_EXPR] in format "==" or "@" + Install/Upgrade package using [PACKAGE_EXPR] in format + "[=|@]" Options: --enable Set the default state of the feature to @@ -8080,7 +8083,13 @@ This command pulls and installs package on SONiC host. *NOTE*: this command requ ``` - Example: ``` - admin@sonic:~$ sudo sonic-package-manager install dhcp-relay==1.0.2 + admin@sonic:~$ sudo sonic-package-manager install dhcp-relay=1.0.2 + ``` + ``` + admin@sonic:~$ sudo sonic-package-manager install dhcp-relay@latest + ``` + ``` + admin@sonic:~$ sudo sonic-package-manager install dhcp-relay@sha256:9780f6d83e45878749497a6297ed9906c19ee0cc48cc88dc63827564bb8768fd ``` ``` admin@sonic:~$ sudo sonic-package-manager install --from-repository azure/sonic-cpu-report:latest @@ -8111,46 +8120,9 @@ This command uninstalls package from SONiC host. *NOTE*: this command requires e admin@sonic:~$ sudo sonic-package-manager uninstall dhcp-relay ``` -**sonic-package-manager upgrade** - -This command upgrades package on SONiC host to a newer version. The procedure of upgrading a package will restart the corresponding service. *NOTE*: this command requires elevated (root) privileges to run. - -- Usage: - ``` - Usage: sonic-package-manager upgrade [OPTIONS] [PACKAGE_EXPR] - - Upgrade package using [PACKAGE_EXPR] in format "==" - - Options: - --from-repository TEXT Fetch package directly from image registry - repository NOTE: This argument is mutually exclusive - with arguments: [package_expr, from_tarball]. - --from-tarball FILE Fetch package from saved image tarball NOTE: This - argument is mutually exclusive with arguments: - [from_repository, package_expr]. - -f, --force Force operation by ignoring failures - -y, --yes Automatically answer yes on prompts - -v, --verbosity LVL Either CRITICAL, ERROR, WARNING, INFO or DEBUG - --skip-host-plugins Do not install host OS plugins provided by the - package (CLI, etc). NOTE: In case when package host - OS plugins are set as mandatory in package manifest - this option will fail the installation. - --help Show this message and exit. - ``` -- Example: - ``` - admin@sonic:~$ sudo sonic-package-manager upgrade dhcp-relay==2.0.0 - ``` - ``` - admin@sonic:~$ sudo sonic-package-manager upgrade --from-repository azure/sonic-cpu-report:latest - ``` - ``` - admin@sonic:~$ sudo sonic-package-manager upgrade --from-tarball sonic-docker-image.gz - ``` - **sonic-package-manager reset** -This comamnd resets the package by reinstalling it to its default version. +This comamnd resets the package by reinstalling it to its default version. *NOTE*: this command requires elevated (root) privileges to run. - Usage: ``` @@ -8195,6 +8167,19 @@ This command will access repository for corresponding package and retrieve a lis • 1.0.2 • 2.0.0 ``` + ``` + admin@sonic:~$ sonic-package-manager show package versions dhcp-relay --plain + 1.0.0 + 1.0.2 + 2.0.0 + ``` + ``` + admin@sonic:~$ sonic-package-manager show package versions dhcp-relay --all + • 1.0.0 + • 1.0.2 + • 2.0.0 + • latest + ``` **sonic-package-manager show package changelog** @@ -8247,7 +8232,7 @@ This command fetches the package manifest and displays it. *NOTE*: package manif ``` - Example: ``` - admin@sonic:~$ sonic-package-manager show package manifest dhcp-relay==2.0.0 + admin@sonic:~$ sonic-package-manager show package manifest dhcp-relay=2.0.0 { "version": "1.0.0", "package": { diff --git a/sonic_package_manager/main.py b/sonic_package_manager/main.py index 13a59e8edb..1f2d5011d4 100644 --- a/sonic_package_manager/main.py +++ b/sonic_package_manager/main.py @@ -174,7 +174,7 @@ def package(ctx): @cli.command() @click.pass_context def list(ctx): - """ List available repositories """ + """ List available packages """ table_header = ['Name', 'Repository', 'Description', 'Version', 'Status'] table_body = [] @@ -341,11 +341,14 @@ def install(ctx, enable, default_owner, skip_host_plugins): - """ Install package using [PACKAGE_EXPR] in format "==" """ + """ Install/Upgrade package using [PACKAGE_EXPR] in format + "[=|@]" """ manager: PackageManager = ctx.obj package_source = package_expr or from_repository or from_tarball + if not package_source: + exit_cli(f'Package source is not specified', fg='red') if not yes and not force: click.confirm(f'{package_source} is going to be installed, ' @@ -369,45 +372,6 @@ def install(ctx, exit_cli(f'Operation canceled by user', fg='red') -@cli.command() -@add_options(PACKAGE_SOURCE_OPTIONS) -@add_options(PACKAGE_COMMON_OPERATION_OPTIONS) -@add_options(PACKAGE_COMMON_INSTALL_OPTIONS) -@click.pass_context -@root_privileges_required -def upgrade(ctx, - package_expr, - from_repository, - from_tarball, - force, - yes, - skip_host_plugins): - """ Upgrade package using [PACKAGE_EXPR] in format "==" """ - - manager: PackageManager = ctx.obj - - package_source = package_expr or from_repository or from_tarball - - if not yes and not force: - click.confirm(f'Package is going to be upgraded with {package_source}, ' - f'continue?', abort=True, show_default=True) - - upgrade_opts = { - 'force': force, - 'skip_host_plugins': skip_host_plugins, - } - - try: - manager.upgrade(package_expr, - from_repository, - from_tarball, - **upgrade_opts) - except Exception as err: - exit_cli(f'Failed to upgrade {package_source}: {err}', fg='red') - except KeyboardInterrupt: - exit_cli(f'Operation canceled by user', fg='red') - - @cli.command() @add_options(PACKAGE_COMMON_OPERATION_OPTIONS) @add_options(PACKAGE_COMMON_INSTALL_OPTIONS) diff --git a/sonic_package_manager/manager.py b/sonic_package_manager/manager.py index 4077f875eb..281f890bcc 100644 --- a/sonic_package_manager/manager.py +++ b/sonic_package_manager/manager.py @@ -4,6 +4,7 @@ import os import pkgutil import tempfile +from inspect import signature from typing import Any, Iterable, Callable, Dict, Optional import docker @@ -37,7 +38,10 @@ from sonic_package_manager.progress import ProgressManager from sonic_package_manager.reference import PackageReference from sonic_package_manager.registry import RegistryResolver -from sonic_package_manager.service_creator.creator import ServiceCreator, run_command +from sonic_package_manager.service_creator.creator import ( + ServiceCreator, + run_command +) from sonic_package_manager.service_creator.feature import FeatureRegistry from sonic_package_manager.service_creator.sonic_db import SonicDB from sonic_package_manager.service_creator.utils import in_chroot @@ -51,7 +55,8 @@ from sonic_package_manager.version import ( Version, VersionRange, - version_to_tag, tag_to_version + version_to_tag, + tag_to_version ) @@ -80,7 +85,23 @@ def wrapped_function(*args, **kwargs): return wrapped_function -def rollback_wrapper(func, *args, **kwargs): +def opt_check(func: Callable) -> Callable: + """ Check kwargs for function. """ + + @functools.wraps(func) + def wrapped_function(*args, **kwargs): + sig = signature(func) + redundant_opts = [opt for opt in kwargs if opt not in sig.parameters] + if redundant_opts: + raise PackageManagerError( + f'Unsupported options: {",".join(redundant_opts)} for {func.__name__}' + ) + return func(*args, **kwargs) + + return wrapped_function + + +def rollback(func, *args, **kwargs): """ Used in rollback callbacks to ignore failure but proceed with rollback. Error will be printed but not fail the whole procedure of rollback. """ @@ -104,7 +125,7 @@ def package_constraint_to_reference(constraint: PackageConstraint) -> PackageRef return PackageReference(package_name, None) if not isinstance(version_constraint, Version): raise PackageManagerError(f'Can only install specific version. ' - f'Use only following expression "{package_name}==" ' + f'Use only following expression "{package_name}=" ' f'to install specific version') return PackageReference(package_name, version_to_tag(version_constraint)) @@ -280,23 +301,29 @@ def install(self, repotag: Optional[str] = None, tarball: Optional[str] = None, **kwargs): - """ Install SONiC Package from either an expression representing - the package and its version, repository and tag or digest in - same format as "docker pulL" accepts or an image tarball path. + """ Install/Upgrade SONiC Package from either an expression + representing the package and its version, repository and tag or + digest in same format as "docker pulL" accepts or an image tarball path. Args: expression: SONiC Package reference expression - repotag: Install from REPO[:TAG][@DIGEST] - tarball: Install from tarball, path to tarball file - kwargs: Install options for self.install_from_source + repotag: Install/Upgrade from REPO[:TAG][@DIGEST] + tarball: Install/Upgrade from tarball, path to tarball file + kwargs: Install/Upgrade options for self.install_from_source Raises: PackageManagerError """ source = self.get_package_source(expression, repotag, tarball) - self.install_from_source(source, **kwargs) + package = source.get_package() + + if self.is_installed(package.name): + self.upgrade_from_source(source, **kwargs) + else: + self.install_from_source(source, **kwargs) @under_lock + @opt_check def install_from_source(self, source: PackageSource, force=False, @@ -340,18 +367,18 @@ def install_from_source(self, self.database.add_package(package.name, package.repository) try: - with contextlib.ExitStack() as exit_stack: + with contextlib.ExitStack() as exits: source.install(package) - exit_stack.callback(rollback_wrapper(source.uninstall, package)) + exits.callback(rollback(source.uninstall, package)) self.service_creator.create(package, state=feature_state, owner=default_owner) - exit_stack.callback(rollback_wrapper(self.service_creator.remove, package)) + exits.callback(rollback(self.service_creator.remove, package)) if not skip_host_plugins: self._install_cli_plugins(package) - exit_stack.callback(rollback_wrapper(self._uninstall_cli_plugins, package)) + exits.callback(rollback(self._uninstall_cli_plugins, package)) - exit_stack.pop_all() + exits.pop_all() except Exception as err: raise PackageInstallationError(f'Failed to install {package.name}: {err}') except KeyboardInterrupt: @@ -363,6 +390,7 @@ def install_from_source(self, self.database.commit() @under_lock + @opt_check def uninstall(self, name: str, force=False): """ Uninstall SONiC Package referenced by name. The uninstallation can be forced if force argument is True. @@ -381,6 +409,8 @@ def uninstall(self, name: str, force=False): package = self.get_installed_package(name) service_name = package.manifest['service']['name'] + name = package.name + with failure_ignore(force): if self.feature_registry.is_feature_enabled(service_name): raise PackageUninstallationError( @@ -402,14 +432,17 @@ def uninstall(self, name: str, force=False): self.service_creator.remove(package) # Clean containers based on this image - containers = self.docker.ps(filters={'ancestor': package.image_id}, all=True) + containers = self.docker.ps(filters={'ancestor': package.image_id}, + all=True) for container in containers: self.docker.rm(container.id, force=True) self.docker.rmi(package.image_id, force=True) package.entry.image_id = None except Exception as err: - raise PackageUninstallationError(f'Failed to uninstall {package.name}: {err}') + raise PackageUninstallationError( + f'Failed to uninstall {package.name}: {err}' + ) package.entry.installed = False package.entry.version = None @@ -417,28 +450,7 @@ def uninstall(self, name: str, force=False): self.database.commit() @under_lock - def upgrade(self, - expression: Optional[str] = None, - repotag: Optional[str] = None, - tarball: Optional[str] = None, - **kwargs): - """ Upgrade SONiC Package from either an expression representing - the package and its version, repository and tag or digest in - same format as "docker pulL" accepts or an image tarball path. - - Args: - expression: SONiC Package reference expression - repotag: Upgrade from REPO[:TAG][@DIGEST] - tarball: Upgrade from tarball, path to tarball file - kwargs: Upgrade options for self.upgrade_from_source - Raises: - PackageManagerError - """ - - source = self.get_package_source(expression, repotag, tarball) - self.upgrade_from_source(source, **kwargs) - - @under_lock + @opt_check def upgrade_from_source(self, source: PackageSource, force=False, @@ -467,7 +479,9 @@ def upgrade_from_source(self, old_package = self.get_installed_package(name) if old_package.built_in: - raise PackageUpgradeError(f'Cannot upgrade built-in package {old_package.name}') + raise PackageUpgradeError( + f'Cannot upgrade built-in package {old_package.name}' + ) old_feature = old_package.manifest['service']['name'] new_feature = new_package.manifest['service']['name'] @@ -483,8 +497,10 @@ def upgrade_from_source(self, # the downgrade might be safe to do. There can be a variable in manifest # describing package downgrade ability or downgrade-able versions. if new_version < old_version and not allow_downgrade: - raise PackageUpgradeError(f'Request to downgrade from {old_version} to {new_version}. ' - f'Downgrade might be not supported by the package') + raise PackageUpgradeError( + f'Request to downgrade from {old_version} to {new_version}. ' + f'Downgrade might be not supported by the package' + ) # remove currently installed package from the list installed_packages = self._get_installed_packages_and(new_package) @@ -497,41 +513,42 @@ def upgrade_from_source(self, # After all checks are passed we proceed to actual upgrade try: - with contextlib.ExitStack() as exit_stack: + with contextlib.ExitStack() as exits: self._uninstall_cli_plugins(old_package) - exit_stack.callback(rollback_wrapper(self._install_cli_plugins, old_package)) + exits.callback(rollback(self._install_cli_plugins, old_package)) source.install(new_package) - exit_stack.callback(rollback_wrapper(source.uninstall, new_package)) + exits.callback(rollback(source.uninstall, new_package)) if self.feature_registry.is_feature_enabled(old_feature): self._systemctl_action(old_package, 'stop') - exit_stack.callback(rollback_wrapper(self._systemctl_action, + exits.callback(rollback(self._systemctl_action, old_package, 'start')) self.service_creator.remove(old_package, deregister_feature=False) - exit_stack.callback(rollback_wrapper(self.service_creator.create, - old_package, register_feature=False)) - - # This is no return point, after we start removing old Docker images - # there is no guaranty we can actually successfully roll-back. + exits.callback(rollback(self.service_creator.create, + old_package, register_feature=False)) # Clean containers based on the old image - containers = self.docker.ps(filters={'ancestor': old_package.image_id}, all=True) + containers = self.docker.ps(filters={'ancestor': old_package.image_id}, + all=True) for container in containers: self.docker.rm(container.id, force=True) - self.docker.rmi(old_package.image_id, force=True) - self.service_creator.create(new_package, register_feature=False) + exits.callback(rollback(self.service_creator.remove, new_package, + register_feature=False)) if self.feature_registry.is_feature_enabled(new_feature): self._systemctl_action(new_package, 'start') if not skip_host_plugins: self._install_cli_plugins(new_package) + exits.callback(rollback(self._uninstall_cli_plugin, old_package)) - exit_stack.pop_all() + self.docker.rmi(old_package.image_id, force=True) + + exits.pop_all() except Exception as err: raise PackageUpgradeError(f'Failed to upgrade {new_package.name}: {err}') except KeyboardInterrupt: @@ -544,6 +561,7 @@ def upgrade_from_source(self, self.database.commit() @under_lock + @opt_check def reset(self, name: str, force: bool = False, skip_host_plugins: bool = False): """ Reset package to defaults version @@ -574,19 +592,18 @@ def reset(self, name: str, force: bool = False, skip_host_plugins: bool = False) def migrate_packages(self, old_package_database: PackageDatabase, dockerd_sock: Optional[str] = None): - """ Migrate packages from old database. This function can - do a comparison between current database and the database - passed in as argument. - If the package is missing in the current database it will be added. - If the package is installed in the passed database and in the current - it is not installed it will be installed with a passed database package version. - If the package is installed in the passed database and it is installed - in the current database but with older version the package will be upgraded to - the never version. - If the package is installed in the passed database and in the current - it is installed but with never version - no actions are taken. - If dockerd_sock parameter is passed, the migration process will use loaded - images from docker library of the currently installed image. + """ + Migrate packages from old database. This function can do a comparison between + current database and the database passed in as argument. If the package is + missing in the current database it will be added. If the package is installed + in the passed database and in the current it is not installed it will be + installed with a passed database package version. If the package is installed + in the passed database and it is installed in the current database but with + older version the package will be upgraded to the never version. If the package + is installed in the passed database and in the current it is installed but with + never version - no actions are taken. If dockerd_sock parameter is passed, the + migration process will use loaded images from docker library of the currently + installed image. Args: old_package_database: SONiC Package Database to migrate packages from. @@ -598,31 +615,21 @@ def migrate_packages(self, self._migrate_package_database(old_package_database) def migrate_package(old_package_entry, - new_package_entry, - migrate_operation=None): + new_package_entry): """ Migrate package routine Args: old_package_entry: Entry in old package database. new_package_entry: Entry in new package database. - migrate_operation: Operation to perform: install or upgrade. """ - try: - migrate_func = { - 'install': self.install, - 'upgrade': self.upgrade, - }[migrate_operation] - except KeyError: - raise ValueError(f'Invalid operation passed in {migrate_operation}') - name = new_package_entry.name version = new_package_entry.version if dockerd_sock: # dockerd_sock is defined, so use docked_sock to connect to # dockerd and fetch package image from it. - log.info(f'{migrate_operation} {name} from old docker library') + log.info(f'installing {name} from old docker library') docker_api = DockerApi(docker.DockerClient(base_url=f'unix://{dockerd_sock}')) image = docker_api.get_image(old_package_entry.image_id) @@ -631,11 +638,11 @@ def migrate_package(old_package_entry, for chunk in image.save(named=True): file.write(chunk) - migrate_func(tarball=file.name) + self.install(tarball=file.name) else: - log.info(f'{migrate_operation} {name} version {version}') + log.info(f'installing {name} version {version}') - migrate_func(f'{name}=={version}') + self.install(f'{name}={version}') # TODO: Topological sort packages by their dependencies first. for old_package in old_package_database: @@ -653,7 +660,7 @@ def migrate_package(old_package_entry, f'{old_package.version} > {new_package.version}') log.info(f'upgrading {new_package.name} to {old_package.version}') new_package.version = old_package.version - migrate_package(old_package, new_package, 'upgrade') + migrate_package(old_package, new_package) else: log.info(f'skipping {new_package.name} as installed version is newer') elif new_package.default_reference is not None: @@ -666,14 +673,14 @@ def migrate_package(old_package_entry, f'then the default in new image: ' f'{old_package.version} > {new_package_default_version}') new_package.version = old_package.version - migrate_package(old_package, new_package, 'install') + migrate_package(old_package, new_package) else: - self.install(f'{new_package.name}=={new_package_default_version}') + self.install(f'{new_package.name}={new_package_default_version}') else: # No default version and package is not installed. # Migrate old package same version. new_package.version = old_package.version - migrate_package(old_package, new_package, 'install') + migrate_package(old_package, new_package) self.database.commit() diff --git a/sonic_package_manager/source.py b/sonic_package_manager/source.py index 40d2408ba7..c179e0b3ee 100644 --- a/sonic_package_manager/source.py +++ b/sonic_package_manager/source.py @@ -28,10 +28,12 @@ def get_metadata(self) -> Metadata: """ raise NotImplementedError - def install_image(self): + def install_image(self, package: Package): """ Install image based on package source. Child class has to implement this method. + Args: + package: SONiC Package Returns: Docker Image object. """ @@ -46,7 +48,7 @@ def install(self, package: Package): package: SONiC Package """ - image = self.install_image() + image = self.install_image(package) package.entry.image_id = image.id # if no repository is defined for this package # get repository from image @@ -113,7 +115,7 @@ def get_metadata(self) -> Metadata: return self.metadata_resolver.from_tarball(self.tarball_path) - def install_image(self): + def install_image(self, package: Package): """ Installs image from local tarball source. """ return self.docker.load(self.tarball_path) @@ -141,10 +143,13 @@ def get_metadata(self) -> Metadata: return self.metadata_resolver.from_registry(self.repository, self.reference) - def install_image(self): + def install_image(self, package: Package): """ Installs image from registry. """ - return self.docker.pull(self.repository, self.reference) + image_id = self.docker.pull(self.repository, self.reference) + if not package.entry.default_reference: + package.entry.default_reference = self.reference + return image_id class LocalSource(PackageSource): diff --git a/tests/sonic_package_manager/test_manager.py b/tests/sonic_package_manager/test_manager.py index 08b957475a..dc79f7f483 100644 --- a/tests/sonic_package_manager/test_manager.py +++ b/tests/sonic_package_manager/test_manager.py @@ -9,12 +9,16 @@ def test_installation_not_installed(package_manager): package_manager.install('test-package') + package = package_manager.get_installed_package('test-package') + assert package.installed + assert package.entry.default_reference == '1.6.0' def test_installation_already_installed(package_manager): - with pytest.raises(PackageInstallationError, - match='swss is already installed'): - package_manager.install('swss') + package_manager.install('test-package') + with pytest.raises(PackageManagerError, + match='1.6.0 is already installed'): + package_manager.install('test-package') def test_installation_dependencies(package_manager, fake_metadata_resolver, mock_docker_api): @@ -197,7 +201,7 @@ def test_installation_using_reference(package_manager, def test_manager_installation_tag(package_manager, mock_docker_api, anything): - package_manager.install(f'test-package==1.6.0') + package_manager.install(f'test-package=1.6.0') mock_docker_api.pull.assert_called_once_with('Azure/docker-test', '1.6.0') @@ -240,10 +244,10 @@ def test_installation_from_file_unknown_package(package_manager, fake_db, sonic_ def test_upgrade_from_file_known_package(package_manager, fake_db, sonic_fs): repository = fake_db.get_package('test-package-6').repository # install older version from repository - package_manager.install('test-package-6==1.5.0') + package_manager.install('test-package-6=1.5.0') # upgrade from file sonic_fs.create_file('Azure/docker-test-6:2.0.0') - package_manager.upgrade(tarball='Azure/docker-test-6:2.0.0') + package_manager.install(tarball='Azure/docker-test-6:2.0.0') # locally installed package does not override already known package repository assert repository == fake_db.get_package('test-package-6').repository @@ -272,22 +276,24 @@ def test_installation_fault(package_manager, mock_docker_api, mock_service_creat def test_manager_installation_version_range(package_manager): with pytest.raises(PackageManagerError, match='Can only install specific version. ' - 'Use only following expression "test-package==" ' + 'Use only following expression "test-package=" ' 'to install specific version'): package_manager.install(f'test-package>=1.6.0') def test_manager_upgrade(package_manager, sonic_fs): - package_manager.install('test-package-6==1.5.0') - package_manager.upgrade('test-package-6==2.0.0') + package_manager.install('test-package-6=1.5.0') + package = package_manager.get_installed_package('test-package-6') + package_manager.install('test-package-6=2.0.0') upgraded_package = package_manager.get_installed_package('test-package-6') assert upgraded_package.entry.version == Version(2, 0, 0) + assert upgraded_package.entry.default_reference == package.entry.default_reference def test_manager_package_reset(package_manager, sonic_fs): - package_manager.install('test-package-6==1.5.0') - package_manager.upgrade('test-package-6==2.0.0') + package_manager.install('test-package-6=1.5.0') + package_manager.install('test-package-6=2.0.0') package_manager.reset('test-package-6') upgraded_package = package_manager.get_installed_package('test-package-6') @@ -296,22 +302,20 @@ def test_manager_package_reset(package_manager, sonic_fs): def test_manager_migration(package_manager, fake_db_for_migration): package_manager.install = Mock() - package_manager.upgrade = Mock() package_manager.migrate_packages(fake_db_for_migration) - # test-package-3 was installed but there is a newer version installed - # in fake_db_for_migration, asserting for upgrade - package_manager.upgrade.assert_has_calls([call('test-package-3==1.6.0')], any_order=True) - package_manager.install.assert_has_calls([ + # test-package-3 was installed but there is a newer version installed + # in fake_db_for_migration, asserting for upgrade + call('test-package-3=1.6.0'), # test-package-4 was not present in DB at all, but it is present and installed in # fake_db_for_migration, thus asserting that it is going to be installed. - call('test-package-4==1.5.0'), + call('test-package-4=1.5.0'), # test-package-5 1.5.0 was installed in fake_db_for_migration but the default # in current db is 1.9.0, assert that migration will install the newer version. - call('test-package-5==1.9.0'), + call('test-package-5=1.9.0'), # test-package-6 2.0.0 was installed in fake_db_for_migration but the default # in current db is 1.5.0, assert that migration will install the newer version. - call('test-package-6==2.0.0')], + call('test-package-6=2.0.0')], any_order=True ) From 180eae301434f004756d8c0772f32b70f18ee933 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Tue, 20 Apr 2021 19:26:19 +0300 Subject: [PATCH 034/173] remove config when uninstalling Signed-off-by: Stepan Blyschak --- .../service_creator/creator.py | 48 +++++++++++++------ .../test_service_creator.py | 4 +- 2 files changed, 36 insertions(+), 16 deletions(-) diff --git a/sonic_package_manager/service_creator/creator.py b/sonic_package_manager/service_creator/creator.py index 53fa51075d..d86b70f979 100644 --- a/sonic_package_manager/service_creator/creator.py +++ b/sonic_package_manager/service_creator/creator.py @@ -146,6 +146,7 @@ def remove_file(path): if deregister_feature: self.feature_registry.deregister(package.manifest['service']['name']) + self.remove_config(package) def post_operation_hook(self): if not in_chroot(): @@ -287,31 +288,31 @@ def generate_dump_script(self, package): render_template(scrip_template, script_path, render_ctx, executable=True) log.info(f'generated {script_path}') - def set_initial_config(self, package): - init_cfg = package.manifest['package']['init-cfg'] + def get_tables(self, table_name): + tables = [] - def get_tables(table_name): - tables = [] + running_table = self.sonic_db.running_table(table_name) + if running_table is not None: + tables.append(running_table) - running_table = self.sonic_db.running_table(table_name) - if running_table is not None: - tables.append(running_table) + persistent_table = self.sonic_db.persistent_table(table_name) + if persistent_table is not None: + tables.append(persistent_table) - persistent_table = self.sonic_db.persistent_table(table_name) - if persistent_table is not None: - tables.append(persistent_table) + initial_table = self.sonic_db.initial_table(table_name) + if initial_table is not None: + tables.append(initial_table) - initial_table = self.sonic_db.initial_table(table_name) - if initial_table is not None: - tables.append(initial_table) + return tables - return tables + def set_initial_config(self, package): + init_cfg = package.manifest['package']['init-cfg'] for tablename, content in init_cfg.items(): if not isinstance(content, dict): continue - tables = get_tables(tablename) + tables = self.get_tables(tablename) for key in content: for table in tables: @@ -321,3 +322,20 @@ def get_tables(table_name): cfg.update(old_fvs) fvs = list(cfg.items()) table.set(key, fvs) + + def remove_config(self, package): + # Remove configuration based on init-cfg tables, so having + # init-cfg even with tables without keys might be a good idea. + # TODO: init-cfg should be validated with yang model + # TODO: remove config from tables known to yang model + init_cfg = package.manifest['package']['init-cfg'] + + for tablename, content in init_cfg.items(): + if not isinstance(content, dict): + continue + + tables = self.get_tables(tablename) + + for key in content: + for table in tables: + table._del(key) diff --git a/tests/sonic_package_manager/test_service_creator.py b/tests/sonic_package_manager/test_service_creator.py index f6ef317d8c..c540259b49 100644 --- a/tests/sonic_package_manager/test_service_creator.py +++ b/tests/sonic_package_manager/test_service_creator.py @@ -105,10 +105,12 @@ def test_service_creator_initial_config(sonic_fs, manifest, mock_feature_registr package = Package(entry, Metadata(manifest)) creator.create(package) - mock_table.set.assert_called_with('key_a', [('field_1', 'value_1'), ('field_2', 'original_value_2')]) + creator.remove(package) + mock_table._del.assert_called_with('key_a') + def test_feature_registration(mock_sonic_db, manifest): mock_feature_table = Mock() From 3b984891640073be70f2bab6c0aa38010b7f4803 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Tue, 20 Apr 2021 19:26:28 +0300 Subject: [PATCH 035/173] fix lgtm error Signed-off-by: Stepan Blyschak --- sonic_package_manager/manager.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/sonic_package_manager/manager.py b/sonic_package_manager/manager.py index 281f890bcc..2a6ea96b9c 100644 --- a/sonic_package_manager/manager.py +++ b/sonic_package_manager/manager.py @@ -409,8 +409,6 @@ def uninstall(self, name: str, force=False): package = self.get_installed_package(name) service_name = package.manifest['service']['name'] - name = package.name - with failure_ignore(force): if self.feature_registry.is_feature_enabled(service_name): raise PackageUninstallationError( From 3280b5b32d0195dda06d0fcf309b43dbe37c0ec9 Mon Sep 17 00:00:00 2001 From: Stepan Blyshchak <38952541+stepanblyschak@users.noreply.github.com> Date: Wed, 21 Apr 2021 03:55:58 +0300 Subject: [PATCH 036/173] Update doc/Command-Reference.md Co-authored-by: Jan Klare --- doc/Command-Reference.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/Command-Reference.md b/doc/Command-Reference.md index 01ab77acce..7caa0d891b 100644 --- a/doc/Command-Reference.md +++ b/doc/Command-Reference.md @@ -7962,7 +7962,7 @@ Go Back To [Beginning of the document](#) or [Beginning of this section](#waterm ## Software Installation and Management -SONiC image can be installed in one of two methods: +SONiC images can be installed in one of two methods: 1. From within a running SONiC image using the `sonic-installer` utility 2. From the vendor's bootloader (E.g., ONIE, Aboot, etc.) From 21ae66be5553ff2ff02caa7c0c734b44936e7592 Mon Sep 17 00:00:00 2001 From: Stepan Blyshchak <38952541+stepanblyschak@users.noreply.github.com> Date: Wed, 21 Apr 2021 03:56:13 +0300 Subject: [PATCH 037/173] Update doc/Command-Reference.md Co-authored-by: Jan Klare --- doc/Command-Reference.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/Command-Reference.md b/doc/Command-Reference.md index 7caa0d891b..750c492f0a 100644 --- a/doc/Command-Reference.md +++ b/doc/Command-Reference.md @@ -7966,7 +7966,7 @@ SONiC images can be installed in one of two methods: 1. From within a running SONiC image using the `sonic-installer` utility 2. From the vendor's bootloader (E.g., ONIE, Aboot, etc.) -SONiC feature Docker images (aka "SONiC packages") available to be installed with *sonic-package-manager* utility. +SONiC packages are available as prebuilt Docker images and meant to be installed with the *sonic-package-manager* utility. ### SONiC Package Manager From 55489765d959787e3d04383e91f3d28a8446051d Mon Sep 17 00:00:00 2001 From: Stepan Blyshchak <38952541+stepanblyschak@users.noreply.github.com> Date: Wed, 21 Apr 2021 03:56:35 +0300 Subject: [PATCH 038/173] Update doc/Command-Reference.md Co-authored-by: Jan Klare --- doc/Command-Reference.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/Command-Reference.md b/doc/Command-Reference.md index 750c492f0a..466ae36dfc 100644 --- a/doc/Command-Reference.md +++ b/doc/Command-Reference.md @@ -7970,7 +7970,7 @@ SONiC packages are available as prebuilt Docker images and meant to be installed ### SONiC Package Manager -This is a command line tool that provides functionality to manage SONiC Packages on SONiC device. +The *sonic-package-manager* is a command line tool to manage (e.g. install, upgrade or uninstall) SONiC Packages. **sonic-package-manager list** From 5a2675dd9c84f8e6a0d75ca8d85dd7d2eb6259f9 Mon Sep 17 00:00:00 2001 From: Stepan Blyshchak <38952541+stepanblyschak@users.noreply.github.com> Date: Wed, 21 Apr 2021 03:57:07 +0300 Subject: [PATCH 039/173] Update doc/Command-Reference.md Co-authored-by: Jan Klare --- doc/Command-Reference.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/Command-Reference.md b/doc/Command-Reference.md index 466ae36dfc..c42f2c5bec 100644 --- a/doc/Command-Reference.md +++ b/doc/Command-Reference.md @@ -8007,7 +8007,7 @@ SONiC package status can be *Installed*, *Not installed* or *Built-In*. "Built-I **sonic-package-manager repository add** -This command will add a new entry in the package database. *NOTE*: this command requires elevated (root) privileges to run. +This command will add a new repository as source for SONiC packages to the database. *NOTE*: requires elevated (root) privileges to run - Usage: ``` From 68f90f5b2c25562cd17e570905a7eec8838e3da0 Mon Sep 17 00:00:00 2001 From: Stepan Blyshchak <38952541+stepanblyschak@users.noreply.github.com> Date: Wed, 21 Apr 2021 03:57:27 +0300 Subject: [PATCH 040/173] Update doc/Command-Reference.md Co-authored-by: Jan Klare --- doc/Command-Reference.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/Command-Reference.md b/doc/Command-Reference.md index c42f2c5bec..634853b8ce 100644 --- a/doc/Command-Reference.md +++ b/doc/Command-Reference.md @@ -8013,7 +8013,7 @@ This command will add a new repository as source for SONiC packages to the datab ``` Usage: sonic-package-manager repository add [OPTIONS] NAME REPOSITORY - Add a new repository to database. Repository in Docker Registry V2. + Add new repository to database Options: --default-reference TEXT Default installation reference From e7f1c598e09418dd0f15b6b13a9906a10d473c53 Mon Sep 17 00:00:00 2001 From: Stepan Blyshchak <38952541+stepanblyschak@users.noreply.github.com> Date: Wed, 21 Apr 2021 03:57:50 +0300 Subject: [PATCH 041/173] Update doc/Command-Reference.md Co-authored-by: Jan Klare --- doc/Command-Reference.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/Command-Reference.md b/doc/Command-Reference.md index 634853b8ce..b47b908771 100644 --- a/doc/Command-Reference.md +++ b/doc/Command-Reference.md @@ -8028,7 +8028,7 @@ This command will add a new repository as source for SONiC packages to the datab **sonic-package-manager repository remove** -This command will remove an entry from the package database. The package has to be *Not Installed* in order to be removed from package database. *NOTE*: this command requires elevated (root) privileges to run. +This command will remove a repository as source for SONiC packages from the database . The package has to be *Not Installed* in order to be removed from package database. *NOTE*: requires elevated (root) privileges to run - Usage: ``` From ae5bad9acb1b0384bf30c7c86c33dff7dff94856 Mon Sep 17 00:00:00 2001 From: Stepan Blyshchak <38952541+stepanblyschak@users.noreply.github.com> Date: Wed, 21 Apr 2021 03:59:17 +0300 Subject: [PATCH 042/173] Update doc/Command-Reference.md Co-authored-by: Jan Klare --- doc/Command-Reference.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/Command-Reference.md b/doc/Command-Reference.md index b47b908771..c837cc3a54 100644 --- a/doc/Command-Reference.md +++ b/doc/Command-Reference.md @@ -8034,7 +8034,7 @@ This command will remove a repository as source for SONiC packages from the data ``` Usage: sonic-package-manager repository remove [OPTIONS] NAME - Remove package from database. + Remove repository from database Options: --help Show this message and exit. From 909079a4ed2fc571335c6095c48e9b4bb0215765 Mon Sep 17 00:00:00 2001 From: Stepan Blyshchak <38952541+stepanblyschak@users.noreply.github.com> Date: Wed, 21 Apr 2021 03:59:44 +0300 Subject: [PATCH 043/173] Update doc/Command-Reference.md Co-authored-by: Jan Klare --- doc/Command-Reference.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/Command-Reference.md b/doc/Command-Reference.md index c837cc3a54..1a3a6bc714 100644 --- a/doc/Command-Reference.md +++ b/doc/Command-Reference.md @@ -8218,7 +8218,7 @@ This command fetches the package manifest and displays it. *NOTE*: package manif ``` Usage: sonic-package-manager show package manifest [OPTIONS] [PACKAGE_EXPR] - Print package manifest content + Show package manifest Options: --from-repository TEXT Fetch package directly from image registry From a394b27df3713cd84e4353f2972a527edf0bad79 Mon Sep 17 00:00:00 2001 From: Stepan Blyshchak <38952541+stepanblyschak@users.noreply.github.com> Date: Wed, 21 Apr 2021 04:00:12 +0300 Subject: [PATCH 044/173] Update doc/Command-Reference.md Co-authored-by: Jan Klare --- doc/Command-Reference.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/Command-Reference.md b/doc/Command-Reference.md index 1a3a6bc714..5fd79c7431 100644 --- a/doc/Command-Reference.md +++ b/doc/Command-Reference.md @@ -8189,7 +8189,7 @@ This command fetches the changelog from package manifest and displays it. *NOTE* ``` Usage: sonic-package-manager show package changelog [OPTIONS] [PACKAGE_EXPR] - Print package changelog + Show package changelog Options: --from-repository TEXT Fetch package directly from image registry From 9653c616a6c75314497eb93c88374222bb75a78a Mon Sep 17 00:00:00 2001 From: Stepan Blyshchak <38952541+stepanblyschak@users.noreply.github.com> Date: Wed, 21 Apr 2021 04:00:44 +0300 Subject: [PATCH 045/173] Update doc/Command-Reference.md Co-authored-by: Jan Klare --- doc/Command-Reference.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/Command-Reference.md b/doc/Command-Reference.md index 5fd79c7431..7ee47fe34f 100644 --- a/doc/Command-Reference.md +++ b/doc/Command-Reference.md @@ -8316,7 +8316,7 @@ This command is used to install a new image on the alternate image partition. T Done ``` -SONiC image installation will install SONiC packages that are installed in currently running SONiC image. In order to perform clean SONiC installation use *--skip-package-migration* option when installing SONiC image: +Installing a new image using the sonic-installer will keep using the packages installed on the currently running SONiC image and automatically migrate those. In order to perform clean SONiC installation use the *--skip-package-migration* option: - Example: ``` From 0e96ddb749b948d399e1548f07bdc42409b3644c Mon Sep 17 00:00:00 2001 From: Stepan Blyshchak <38952541+stepanblyschak@users.noreply.github.com> Date: Wed, 21 Apr 2021 04:01:09 +0300 Subject: [PATCH 046/173] Update doc/Command-Reference.md Co-authored-by: Jan Klare --- doc/Command-Reference.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/Command-Reference.md b/doc/Command-Reference.md index 7ee47fe34f..5e492da2c1 100644 --- a/doc/Command-Reference.md +++ b/doc/Command-Reference.md @@ -8153,7 +8153,7 @@ This command will access repository for corresponding package and retrieve a lis ``` Usage: sonic-package-manager show package versions [OPTIONS] NAME - Print available versions + Show available versions Options: --all Show all available tags in repository From 9090a476b11cf156800bfc27bca32cb9389e4127 Mon Sep 17 00:00:00 2001 From: Stepan Blyshchak <38952541+stepanblyschak@users.noreply.github.com> Date: Wed, 21 Apr 2021 04:01:29 +0300 Subject: [PATCH 047/173] Update doc/Command-Reference.md Co-authored-by: Jan Klare --- doc/Command-Reference.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/Command-Reference.md b/doc/Command-Reference.md index 5e492da2c1..559774562d 100644 --- a/doc/Command-Reference.md +++ b/doc/Command-Reference.md @@ -8183,7 +8183,7 @@ This command will access repository for corresponding package and retrieve a lis **sonic-package-manager show package changelog** -This command fetches the changelog from package manifest and displays it. *NOTE*: package changelog can be retrieved from registry or read from image tarball without installing it. +This command fetches the changelog from the package manifest and displays it. *NOTE*: package changelog can be retrieved from registry or read from image tarball without installing it. - Usage: ``` From b2eeb594534342282b897909fb5ca47cecd30501 Mon Sep 17 00:00:00 2001 From: Stepan Blyshchak <38952541+stepanblyschak@users.noreply.github.com> Date: Wed, 21 Apr 2021 04:04:30 +0300 Subject: [PATCH 048/173] Update doc/Command-Reference.md Co-authored-by: Jan Klare --- doc/Command-Reference.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/Command-Reference.md b/doc/Command-Reference.md index 559774562d..78be0b0488 100644 --- a/doc/Command-Reference.md +++ b/doc/Command-Reference.md @@ -8147,7 +8147,7 @@ This comamnd resets the package by reinstalling it to its default version. *NOTE **sonic-package-manager show package versions** -This command will access repository for corresponding package and retrieve a list of available versions. +This command will retrieve a list of all available versions for the given package from the configured upstream repository - Usage: ``` From e765d5d19bd842f2c0abdb83f5e94cac8108ee3e Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Wed, 21 Apr 2021 04:05:45 +0300 Subject: [PATCH 049/173] review comments Signed-off-by: Stepan Blyschak --- doc/Command-Reference.md | 3 +-- sonic_package_manager/main.py | 12 +++++------- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/doc/Command-Reference.md b/doc/Command-Reference.md index b47b908771..4e4bbc455e 100644 --- a/doc/Command-Reference.md +++ b/doc/Command-Reference.md @@ -8046,8 +8046,7 @@ This command will remove a repository as source for SONiC packages from the data **sonic-package-manager install** -This command pulls and installs or upgrades (if already installed) package on SONiC host. *NOTE*: this command requires elevated (root) privileges to run. -The procedure of upgrading a package will restart the corresponding service. +This command pulls and installs a package on SONiC host. *NOTE*: this command requires elevated (root) privileges to run - Usage: ``` diff --git a/sonic_package_manager/main.py b/sonic_package_manager/main.py index 1f2d5011d4..d29147f809 100644 --- a/sonic_package_manager/main.py +++ b/sonic_package_manager/main.py @@ -208,7 +208,7 @@ def manifest(ctx, package_expr, from_repository, from_tarball): - """ Print package manifest content """ + """ Show package manifest """ manager: PackageManager = ctx.obj @@ -228,7 +228,7 @@ def manifest(ctx, @click.option('--plain', is_flag=True, help='Plain output') @click.pass_context def versions(ctx, name, all, plain): - """ Print available versions """ + """ Show available versions """ try: manager: PackageManager = ctx.obj @@ -248,7 +248,7 @@ def changelog(ctx, package_expr, from_repository, from_tarball): - """ Print package changelog """ + """ Show package changelog """ manager: PackageManager = ctx.obj @@ -286,9 +286,7 @@ def changelog(ctx, @click.pass_context @root_privileges_required def add(ctx, name, repository, default_reference, description): - """ Add a new repository to database. - Repository in Docker Registry V2. - """ + """ Add a new repository to database """ manager: PackageManager = ctx.obj @@ -306,7 +304,7 @@ def add(ctx, name, repository, default_reference, description): @click.pass_context @root_privileges_required def remove(ctx, name): - """ Remove package from database. """ + """ Remove repository from database. """ manager: PackageManager = ctx.obj From fd46c34536d99524fb701206012f280431533810 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Thu, 22 Apr 2021 12:36:47 +0300 Subject: [PATCH 050/173] fix upgrade rollback Signed-off-by: Stepan Blyschak --- sonic_package_manager/manager.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sonic_package_manager/manager.py b/sonic_package_manager/manager.py index 2a6ea96b9c..527869ac73 100644 --- a/sonic_package_manager/manager.py +++ b/sonic_package_manager/manager.py @@ -521,7 +521,7 @@ def upgrade_from_source(self, if self.feature_registry.is_feature_enabled(old_feature): self._systemctl_action(old_package, 'stop') exits.callback(rollback(self._systemctl_action, - old_package, 'start')) + old_package, 'start')) self.service_creator.remove(old_package, deregister_feature=False) exits.callback(rollback(self.service_creator.create, @@ -539,6 +539,8 @@ def upgrade_from_source(self, if self.feature_registry.is_feature_enabled(new_feature): self._systemctl_action(new_package, 'start') + exits.callback(rollback(self._systemctl_action, + new_package, 'stop')) if not skip_host_plugins: self._install_cli_plugins(new_package) From 1348a6ee30794b0292e4a68cc6e67ddeb5f26374 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Thu, 22 Apr 2021 12:48:17 +0300 Subject: [PATCH 051/173] remove trailing spaces Signed-off-by: Stepan Blyschak --- config/config_mgmt.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/config/config_mgmt.py b/config/config_mgmt.py index e5e929c419..45f18b32c1 100644 --- a/config/config_mgmt.py +++ b/config/config_mgmt.py @@ -62,7 +62,7 @@ def __init__(self, source="configDB", debug=False, allowTablesWithoutYang=True): raise Exception('ConfigMgmt Class creation failed') return - + def __init_sonic_yang(self): self.sy = sonic_yang.SonicYang(YANG_DIR, debug=self.DEBUG) # load yang models @@ -218,14 +218,14 @@ def writeConfigDB(self, jDiff): configdb.mod_config(sonic_cfggen.FormatConverter.output_to_db(data)) return - + def add_module(self, yang_module_text, replace_if_exists=False): """ Validate and add new YANG module to the system. Parameters: yang_module_text (str): YANG module string. - + Returns: None """ @@ -248,7 +248,7 @@ def remove_module(self, module_name): Parameters: module_name (str): YANG module name. - + Returns: None """ @@ -262,7 +262,7 @@ def remove_module(self, module_name): except Exception: self.add_module(yang_module_text) raise - + @staticmethod def get_module_name(yang_module_text): """ @@ -270,7 +270,7 @@ def get_module_name(yang_module_text): Parameters: yang_module_text(str): YANG module string. - + Returns: str: Module name """ @@ -278,7 +278,7 @@ def get_module_name(yang_module_text): sy = sonic_yang.SonicYang(YANG_DIR) module = sy.ctx.parse_module_mem(yang_module_text, ly.LYS_IN_YANG) return module.name() - + # End of Class ConfigMgmt From f4449c15f2cb2b142d115c8f22b554f99d32883c Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Thu, 22 Apr 2021 12:49:11 +0300 Subject: [PATCH 052/173] remove trailing spaces Signed-off-by: Stepan Blyschak --- sonic_package_manager/manager.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/sonic_package_manager/manager.py b/sonic_package_manager/manager.py index 25dd93a116..21a3d30025 100644 --- a/sonic_package_manager/manager.py +++ b/sonic_package_manager/manager.py @@ -603,16 +603,16 @@ def migrate_packages(self, old_package_database: PackageDatabase, dockerd_sock: Optional[str] = None): """ - Migrate packages from old database. This function can do a comparison between - current database and the database passed in as argument. If the package is - missing in the current database it will be added. If the package is installed - in the passed database and in the current it is not installed it will be - installed with a passed database package version. If the package is installed - in the passed database and it is installed in the current database but with - older version the package will be upgraded to the never version. If the package - is installed in the passed database and in the current it is installed but with - never version - no actions are taken. If dockerd_sock parameter is passed, the - migration process will use loaded images from docker library of the currently + Migrate packages from old database. This function can do a comparison between + current database and the database passed in as argument. If the package is + missing in the current database it will be added. If the package is installed + in the passed database and in the current it is not installed it will be + installed with a passed database package version. If the package is installed + in the passed database and it is installed in the current database but with + older version the package will be upgraded to the never version. If the package + is installed in the passed database and in the current it is installed but with + never version - no actions are taken. If dockerd_sock parameter is passed, the + migration process will use loaded images from docker library of the currently installed image. Args: @@ -897,7 +897,7 @@ def _get_cli_plugin_path(cls, package: Package, command): raise PackageManagerError(f'Failed to get plugins path for {command} CLI') plugins_pkg_path = os.path.dirname(pkg_loader.path) return os.path.join(plugins_pkg_path, cls._get_cli_plugin_name(package)) - + def _install_yang_module(self, package: Package): self.cfg_mgmt.add_module(package.metadata.yang_module_text) From 8fd00841a14f62c987e8788fad3c97dba59a28aa Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Thu, 22 Apr 2021 12:49:30 +0300 Subject: [PATCH 053/173] remove trailing spaces Signed-off-by: Stepan Blyschak --- sonic_package_manager/metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sonic_package_manager/metadata.py b/sonic_package_manager/metadata.py index c8f902b3c7..45cca67642 100644 --- a/sonic_package_manager/metadata.py +++ b/sonic_package_manager/metadata.py @@ -182,7 +182,7 @@ def from_labels(cls, labels: Dict[str, str]) -> Metadata: components[component] = Version.parse(version) except ValueError as err: raise MetadataError(f'Failed to parse component version: {err}') - + yang_module_text = sonic_metadata.get('yang-module') return Metadata(Manifest.marshal(manifest_dict), components, yang_module_text) From b065eafc5b0cba361f414a4f76975ca946994fc0 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Fri, 23 Apr 2021 12:53:03 +0300 Subject: [PATCH 054/173] yang Signed-off-by: Stepan Blyschak --- sonic_package_manager/manager.py | 23 +------ .../service_creator/creator.py | 62 ++++++++++++++----- .../service_creator/feature.py | 2 +- .../service_creator/sonic_db.py | 2 +- tests/sonic_package_manager/conftest.py | 2 +- .../test_service_creator.py | 20 +++--- 6 files changed, 65 insertions(+), 46 deletions(-) diff --git a/sonic_package_manager/manager.py b/sonic_package_manager/manager.py index 21a3d30025..91de555edd 100644 --- a/sonic_package_manager/manager.py +++ b/sonic_package_manager/manager.py @@ -377,9 +377,6 @@ def install_from_source(self, self.service_creator.create(package, state=feature_state, owner=default_owner) exits.callback(rollback(self.service_creator.remove, package)) - self._install_yang_module(package) - exits.callback(rollback(self._uninstall_yang_module, package)) - if not skip_host_plugins: self._install_cli_plugins(package) exits.callback(rollback(self._uninstall_cli_plugins, package)) @@ -433,7 +430,6 @@ def uninstall(self, name: str, force=False): try: self._uninstall_cli_plugins(package) - self._uninstall_yang_module(package) self.service_creator.remove(package) # Clean containers based on this image @@ -544,9 +540,6 @@ def upgrade_from_source(self, exits.callback(rollback(self.service_creator.remove, new_package, register_feature=False)) - self._upgrade_yang_module(new_package) - exits.callback(rollback(self._upgrade_yang_module(old_package))) - if self.feature_registry.is_feature_enabled(new_feature): self._systemctl_action(new_package, 'start') exits.callback(rollback(self._systemctl_action, @@ -898,16 +891,6 @@ def _get_cli_plugin_path(cls, package: Package, command): plugins_pkg_path = os.path.dirname(pkg_loader.path) return os.path.join(plugins_pkg_path, cls._get_cli_plugin_name(package)) - def _install_yang_module(self, package: Package): - self.cfg_mgmt.add_module(package.metadata.yang_module_text) - - def _upgrade_yang_module(self, package: Package): - self.cfg_mgmt.add_module(package.metadata.yang_module_text, replace_if_exists=True) - - def _uninstall_yang_module(self, package: Package): - module_name = self.cfg_mgmt.get_module_name(package.metadata.yang_module_text) - self.cfg_mgmt.remove_module(module_name) - def _install_cli_plugins(self, package: Package): for command in ('show', 'config', 'clear'): self._install_cli_plugin(package, command) @@ -941,11 +924,11 @@ def get_manager() -> 'PackageManager': docker_api = DockerApi(docker.from_env()) registry_resolver = RegistryResolver() + cfg_mgmt = config_mgmt.ConfigMgmt() return PackageManager(DockerApi(docker.from_env(), ProgressManager()), registry_resolver, PackageDatabase.from_file(), MetadataResolver(docker_api, registry_resolver), - ServiceCreator(FeatureRegistry(SonicDB), SonicDB), + ServiceCreator(FeatureRegistry(SonicDB), SonicDB, cfg_mgmt), device_info, - filelock.FileLock(PACKAGE_MANAGER_LOCK_FILE, timeout=0), - config_mgmt.ConfigMgmt()) + filelock.FileLock(PACKAGE_MANAGER_LOCK_FILE, timeout=0)) diff --git a/sonic_package_manager/service_creator/creator.py b/sonic_package_manager/service_creator/creator.py index d86b70f979..6bea5b14c9 100644 --- a/sonic_package_manager/service_creator/creator.py +++ b/sonic_package_manager/service_creator/creator.py @@ -1,17 +1,22 @@ #!/usr/bin/env python import contextlib +import json import os import stat import subprocess from typing import Dict import jinja2 as jinja2 +from config.config_mgmt import ConfigMgmt from prettyprinter import pformat - from sonic_package_manager.logger import log from sonic_package_manager.package import Package from sonic_package_manager.service_creator import ETC_SONIC_PATH from sonic_package_manager.service_creator.feature import FeatureRegistry +from sonic_package_manager.service_creator.sonic_db import ( + CONFIG_DB_JSON, + INIT_CFG_JSON +) from sonic_package_manager.service_creator.utils import in_chroot SERVICE_FILE_TEMPLATE = 'sonic.service.j2' @@ -99,9 +104,11 @@ class ServiceCreator: """ Creates and registers services in SONiC based on the package manifest. """ - def __init__(self, feature_registry: FeatureRegistry, sonic_db): + def __init__(self, feature_registry: FeatureRegistry, + sonic_db, cfg_mgmt: ConfigMgmt): self.feature_registry = feature_registry self.sonic_db = sonic_db + self.cfg_mgmt = cfg_mgmt def create(self, package: Package, @@ -114,9 +121,9 @@ def create(self, self.update_dependent_list_file(package) self.generate_systemd_service(package) self.generate_dump_script(package) + self.install_yang_module(package) self.set_initial_config(package) - self.post_operation_hook() if register_feature: @@ -126,7 +133,9 @@ def create(self, self.remove(package, not register_feature) raise - def remove(self, package: Package, deregister_feature=True): + def remove(self, package: Package, + deregister_feature=True, + keep_config=False): name = package.manifest['service']['name'] def remove_file(path): @@ -139,6 +148,7 @@ def remove_file(path): remove_file(os.path.join(SERVICE_MGMT_SCRIPT_LOCATION, f'{name}.sh')) remove_file(os.path.join(DOCKER_CTL_SCRIPT_LOCATION, f'{name}.sh')) remove_file(os.path.join(DEBUG_DUMP_SCRIPT_LOCATION, f'{name}')) + self.uninstall_yang_module(package) self.update_dependent_list_file(package, remove=True) @@ -146,6 +156,8 @@ def remove_file(path): if deregister_feature: self.feature_registry.deregister(package.manifest['service']['name']) + + if deregister_feature and not keep_config: self.remove_config(package) def post_operation_hook(self): @@ -307,6 +319,8 @@ def get_tables(self, table_name): def set_initial_config(self, package): init_cfg = package.manifest['package']['init-cfg'] + if not init_cfg: + return for tablename, content in init_cfg.items(): if not isinstance(content, dict): @@ -322,20 +336,38 @@ def set_initial_config(self, package): cfg.update(old_fvs) fvs = list(cfg.items()) table.set(key, fvs) + + self.validate_configs() def remove_config(self, package): - # Remove configuration based on init-cfg tables, so having - # init-cfg even with tables without keys might be a good idea. - # TODO: init-cfg should be validated with yang model - # TODO: remove config from tables known to yang model - init_cfg = package.manifest['package']['init-cfg'] - - for tablename, content in init_cfg.items(): - if not isinstance(content, dict): + module_name = self.cfg_mgmt.get_module_name(package.metadata.yang_module_text) + for tablename, module in self.cfg_mgmt.sy.confDbYangMap.items(): + if module['module'] != module_name: continue tables = self.get_tables(tablename) - - for key in content: - for table in tables: + for table in tables: + for key in table.getKeys(): table._del(key) + + def validate_configs(self): + """ Validate configuration through YANG """ + + log.debug('validating running configuration') + self.cfg_mgmt.readConfigDB() + self.cfg_mgmt.loadData(self.cfg_mgmt.configdbJsonIn) + + log.debug('validating saved configuration') + self.cfg_mgmt.readConfigDBJson(CONFIG_DB_JSON) + self.cfg_mgmt.loadData(self.cfg_mgmt.configdbJsonIn) + + log.debug('validating initial configuration') + self.cfg_mgmt.readConfigDBJson(INIT_CFG_JSON) + self.cfg_mgmt.loadData(self.cfg_mgmt.configdbJsonIn) + + def install_yang_module(self, package: Package): + self.cfg_mgmt.add_module(package.metadata.yang_module_text) + + def uninstall_yang_module(self, package: Package): + module_name = self.cfg_mgmt.get_module_name(package.metadata.yang_module_text) + self.cfg_mgmt.remove_module(module_name) diff --git a/sonic_package_manager/service_creator/feature.py b/sonic_package_manager/service_creator/feature.py index 4df06384d2..229531c9b5 100644 --- a/sonic_package_manager/service_creator/feature.py +++ b/sonic_package_manager/service_creator/feature.py @@ -64,7 +64,7 @@ def is_feature_enabled(self, name: str) -> bool: def get_multi_instance_features(self): res = [] init_db_table = self._sonic_db.initial_table(FEATURE) - for feature in init_db_table.keys(): + for feature in init_db_table.getKeys(): exists, cfg = init_db_table.get(feature) assert exists cfg = dict(cfg) diff --git a/sonic_package_manager/service_creator/sonic_db.py b/sonic_package_manager/service_creator/sonic_db.py index a9ba837ab1..8f3be792cc 100644 --- a/sonic_package_manager/service_creator/sonic_db.py +++ b/sonic_package_manager/service_creator/sonic_db.py @@ -20,7 +20,7 @@ def __init__(self, file, table): self._file = file self._table = table - def keys(self): + def getKeys(self): with open(self._file) as stream: config = json.load(stream) return config.get(self._table, {}).keys() diff --git a/tests/sonic_package_manager/conftest.py b/tests/sonic_package_manager/conftest.py index 088fde27f1..e3a3f5d458 100644 --- a/tests/sonic_package_manager/conftest.py +++ b/tests/sonic_package_manager/conftest.py @@ -67,7 +67,7 @@ def mock_sonic_db(): @pytest.fixture def mock_config_mgmt(): - yield Mock() + yield MagicMock() @pytest.fixture diff --git a/tests/sonic_package_manager/test_service_creator.py b/tests/sonic_package_manager/test_service_creator.py index c540259b49..da2fa26e49 100644 --- a/tests/sonic_package_manager/test_service_creator.py +++ b/tests/sonic_package_manager/test_service_creator.py @@ -37,8 +37,9 @@ def manifest(): }) -def test_service_creator(sonic_fs, manifest, mock_feature_registry, mock_sonic_db): - creator = ServiceCreator(mock_feature_registry, mock_sonic_db) +def test_service_creator(sonic_fs, manifest, mock_feature_registry, + mock_sonic_db, mock_config_mgmt): + creator = ServiceCreator(mock_feature_registry, mock_sonic_db, mock_config_mgmt) entry = PackageEntry('test', 'azure/sonic-test') package = Package(entry, Metadata(manifest)) creator.create(package) @@ -49,8 +50,9 @@ def test_service_creator(sonic_fs, manifest, mock_feature_registry, mock_sonic_d assert sonic_fs.exists(os.path.join(SYSTEMD_LOCATION, 'test.service')) -def test_service_creator_with_timer_unit(sonic_fs, manifest, mock_feature_registry, mock_sonic_db): - creator = ServiceCreator(mock_feature_registry, mock_sonic_db) +def test_service_creator_with_timer_unit(sonic_fs, manifest, mock_feature_registry, + mock_sonic_db, mock_config_mgmt): + creator = ServiceCreator(mock_feature_registry, mock_sonic_db, mock_config_mgmt) entry = PackageEntry('test', 'azure/sonic-test') package = Package(entry, Metadata(manifest)) creator.create(package) @@ -64,8 +66,9 @@ def test_service_creator_with_timer_unit(sonic_fs, manifest, mock_feature_regist assert sonic_fs.exists(os.path.join(SYSTEMD_LOCATION, 'test.timer')) -def test_service_creator_with_debug_dump(sonic_fs, manifest, mock_feature_registry, mock_sonic_db): - creator = ServiceCreator(mock_feature_registry, mock_sonic_db) +def test_service_creator_with_debug_dump(sonic_fs, manifest, mock_feature_registry, + mock_sonic_db, mock_config_mgmt): + creator = ServiceCreator(mock_feature_registry, mock_sonic_db, mock_config_mgmt) entry = PackageEntry('test', 'azure/sonic-test') package = Package(entry, Metadata(manifest)) creator.create(package) @@ -79,14 +82,15 @@ def test_service_creator_with_debug_dump(sonic_fs, manifest, mock_feature_regist assert sonic_fs.exists(os.path.join(DEBUG_DUMP_SCRIPT_LOCATION, 'test')) -def test_service_creator_initial_config(sonic_fs, manifest, mock_feature_registry, mock_sonic_db): +def test_service_creator_initial_config(sonic_fs, manifest, mock_feature_registry, + mock_sonic_db, mock_config_mgmt): mock_table = Mock() mock_table.get = Mock(return_value=(True, (('field_2', 'original_value_2'),))) mock_sonic_db.initial_table = Mock(return_value=mock_table) mock_sonic_db.persistent_table = Mock(return_value=mock_table) mock_sonic_db.running_table = Mock(return_value=mock_table) - creator = ServiceCreator(mock_feature_registry, mock_sonic_db) + creator = ServiceCreator(mock_feature_registry, mock_sonic_db, mock_config_mgmt) entry = PackageEntry('test', 'azure/sonic-test') package = Package(entry, Metadata(manifest)) From 86bf53f4b7a79383ecd9921372a568d18277a9b9 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Fri, 23 Apr 2021 14:29:07 +0300 Subject: [PATCH 055/173] address review comments Signed-off-by: Stepan Blyschak --- doc/Command-Reference.md | 113 +++++++++++++++++++------------ sonic_installer/main.py | 10 +-- sonic_package_manager/main.py | 78 ++++++++++++--------- sonic_package_manager/manager.py | 6 +- 4 files changed, 122 insertions(+), 85 deletions(-) diff --git a/doc/Command-Reference.md b/doc/Command-Reference.md index 8c7fe7bad2..ab4e28dbdc 100644 --- a/doc/Command-Reference.md +++ b/doc/Command-Reference.md @@ -8013,11 +8013,14 @@ This command will add a new repository as source for SONiC packages to the datab ``` Usage: sonic-package-manager repository add [OPTIONS] NAME REPOSITORY - Add new repository to database + Add a new repository to database. + + NOTE: This command requires elevated (root) privileges to run. Options: - --default-reference TEXT Default installation reference - --description TEXT Optional package entry description + --default-reference TEXT Default installation reference. Can be a tag or + sha256 digest in repository. + --description TEXT Optional package entry description. --help Show this message and exit. ``` - Example: @@ -8034,7 +8037,9 @@ This command will remove a repository as source for SONiC packages from the data ``` Usage: sonic-package-manager repository remove [OPTIONS] NAME - Remove repository from database + Remove repository from database. + + NOTE: This command requires elevated (root) privileges to run. Options: --help Show this message and exit. @@ -8053,32 +8058,46 @@ This command pulls and installs a package on SONiC host. *NOTE*: this command re Usage: sonic-package-manager install [OPTIONS] [PACKAGE_EXPR] Install/Upgrade package using [PACKAGE_EXPR] in format - "[=|@]" + "[=|@]". + + The repository to pull the package from is resolved by lookup in + package database, thus the package has to be added via "sonic- + package-manager repository add" command. + + In case when [PACKAGE_EXPR] is a package name "" this command + will install or upgrade to a version referenced by "default- + reference" in package database. + + NOTE: This command requires elevated (root) privileges to run. Options: - --enable Set the default state of the feature to - enabled and enable feature right after - installation. NOTE: user needs to execute - "config save -y" to make this setting - persistent - --default-owner [local|kube] Default owner configuration setting for a - feature - --from-repository TEXT Fetch package directly from image registry - repository NOTE: This argument is mutually - exclusive with arguments: [package_expr, - from_tarball]. - --from-tarball FILE Fetch package from saved image tarball NOTE: - This argument is mutually exclusive with - arguments: [package_expr, from_repository]. - -f, --force Force operation by ignoring failures - -y, --yes Automatically answer yes on prompts - -v, --verbosity LVL Either CRITICAL, ERROR, WARNING, INFO or DEBUG - --skip-host-plugins Do not install host OS plugins provided by the - package (CLI, etc). NOTE: In case when package - host OS plugins are set as mandatory in - package manifest this option will fail the - installation. - --help Show this message and exit. + --enable Set the default state of the feature to enabled + and enable feature right after installation. NOTE: + user needs to execute "config save -y" to make + this setting persistent. + --set-owner [local|kube] Default owner configuration setting for a feature. + --from-repository TEXT Fetch package directly from image registry + repository. NOTE: This argument is mutually + exclusive with arguments: [package_expr, + from_tarball]. + --from-tarball FILE Fetch package from saved image tarball. NOTE: This + argument is mutually exclusive with arguments: + [package_expr, from_repository]. + -f, --force Force operation by ignoring package dependency + tree and package manifest validation failures. + -y, --yes Automatically answer yes on prompts. + -v, --verbosity LVL Either CRITICAL, ERROR, WARNING, INFO or DEBUG. + Default is INFO. + --skip-host-plugins Do not install host OS plugins provided by the + package (CLI, etc). NOTE: In case when package + host OS plugins are set as mandatory in package + manifest this option will fail the installation. + --allow-downgrade Allow package downgrade. By default an attempt to + downgrade the package will result in a failure + since downgrade might not be supported by the + package, thus requires explicit request from the + user. + --help Show this message and exit.. ``` - Example: ``` @@ -8099,20 +8118,24 @@ This command pulls and installs a package on SONiC host. *NOTE*: this command re **sonic-package-manager uninstall** -This command uninstalls package from SONiC host. *NOTE*: this command requires elevated (root) privileges to run. +This command uninstalls package from SONiC host. User needs to stop the feature prior to uninstalling it. +*NOTE*: this command requires elevated (root) privileges to run. - Usage: ``` Usage: sonic-package-manager uninstall [OPTIONS] NAME - Uninstall package + Uninstall package. + + NOTE: This command requires elevated (root) privileges to run. Options: - -f, --force Force operation by ignoring failures - -y, --yes Automatically answer yes on prompts - -v, --verbosity LVL Either CRITICAL, ERROR, WARNING, INFO or DEBUG + -f, --force Force operation by ignoring package dependency tree and + package manifest validation failures. + -y, --yes Automatically answer yes on prompts. + -v, --verbosity LVL Either CRITICAL, ERROR, WARNING, INFO or DEBUG. Default + is INFO. --help Show this message and exit. - ``` - Example: ``` @@ -8127,12 +8150,16 @@ This comamnd resets the package by reinstalling it to its default version. *NOTE ``` Usage: sonic-package-manager reset [OPTIONS] NAME - Reset package to the default version + Reset package to the default version. + + NOTE: This command requires elevated (root) privileges to run. Options: - -f, --force Force operation by ignoring failures - -y, --yes Automatically answer yes on prompts - -v, --verbosity LVL Either CRITICAL, ERROR, WARNING, INFO or DEBUG + -f, --force Force operation by ignoring package dependency tree and + package manifest validation failures. + -y, --yes Automatically answer yes on prompts. + -v, --verbosity LVL Either CRITICAL, ERROR, WARNING, INFO or DEBUG. Default + is INFO. --skip-host-plugins Do not install host OS plugins provided by the package (CLI, etc). NOTE: In case when package host OS plugins are set as mandatory in package manifest this option @@ -8152,11 +8179,11 @@ This command will retrieve a list of all available versions for the given packag ``` Usage: sonic-package-manager show package versions [OPTIONS] NAME - Show available versions + Show available versions. Options: - --all Show all available tags in repository - --plain Plain output + --all Show all available tags in repository. + --plain Plain output. --help Show this message and exit. ``` - Example: @@ -8188,7 +8215,7 @@ This command fetches the changelog from the package manifest and displays it. *N ``` Usage: sonic-package-manager show package changelog [OPTIONS] [PACKAGE_EXPR] - Show package changelog + Show package changelog. Options: --from-repository TEXT Fetch package directly from image registry @@ -8217,7 +8244,7 @@ This command fetches the package manifest and displays it. *NOTE*: package manif ``` Usage: sonic-package-manager show package manifest [OPTIONS] [PACKAGE_EXPR] - Show package manifest + Show package manifest. Options: --from-repository TEXT Fetch package directly from image registry diff --git a/sonic_installer/main.py b/sonic_installer/main.py index bc4e7dd338..12a2ab7e0e 100644 --- a/sonic_installer/main.py +++ b/sonic_installer/main.py @@ -233,14 +233,14 @@ def mount_squash_fs(squashfs_path, mount_point): def umount(mount_point, read_only=True, recursive=False, force=True, remove_dir=True): - flags = "-" + flags = [] if read_only: - flags = flags + "r" + flags.append("-r") if force: - flags = flags + "f" + flags.append("-f") if recursive: - flags = flags + "R" - run_command_or_raise(["umount", flags, mount_point]) + flags.append("-R") + run_command_or_raise(["umount", *flags, mount_point]) if remove_dir: run_command_or_raise(["rm", "-rf", mount_point]) diff --git a/sonic_package_manager/main.py b/sonic_package_manager/main.py index d29147f809..70e1e67310 100644 --- a/sonic_package_manager/main.py +++ b/sonic_package_manager/main.py @@ -46,6 +46,8 @@ def wrapped_function(*args, **kwargs): return func(*args, **kwargs) + wrapped_function.__doc__ += '\n\n NOTE: This command requires elevated (root) privileges to run.' + return wrapped_function @@ -91,7 +93,7 @@ def handle_parse_result(self, ctx, opts, args): PACKAGE_SOURCE_OPTIONS = [ click.option('--from-repository', - help='Fetch package directly from image registry repository', + help='Fetch package directly from image registry repository.', cls=MutuallyExclusiveOption, mutually_exclusive=['from_tarball', 'package_expr']), click.option('--from-tarball', @@ -99,7 +101,7 @@ def handle_parse_result(self, ctx, opts, args): readable=True, file_okay=True, dir_okay=False), - help='Fetch package from saved image tarball', + help='Fetch package from saved image tarball.', cls=MutuallyExclusiveOption, mutually_exclusive=['from_repository', 'package_expr']), click.argument('package-expr', @@ -110,21 +112,21 @@ def handle_parse_result(self, ctx, opts, args): PACKAGE_COMMON_INSTALL_OPTIONS = [ click.option('--skip-host-plugins', - is_flag=True, - help='Do not install host OS plugins provided by the package (CLI, etc). ' - 'NOTE: In case when package host OS plugins are set as mandatory in ' - 'package manifest this option will fail the installation.'), + is_flag=True, + help='Do not install host OS plugins provided by the package (CLI, etc). ' + 'NOTE: In case when package host OS plugins are set as mandatory in ' + 'package manifest this option will fail the installation.') ] PACKAGE_COMMON_OPERATION_OPTIONS = [ click.option('-f', '--force', is_flag=True, - help='Force operation by ignoring failures'), + help='Force operation by ignoring package dependency tree and package manifest validation failures.'), click.option('-y', '--yes', is_flag=True, - help='Automatically answer yes on prompts'), - click_log.simple_verbosity_option(log), + help='Automatically answer yes on prompts.'), + click_log.simple_verbosity_option(log, help='Either CRITICAL, ERROR, WARNING, INFO or DEBUG. Default is INFO.'), ] @@ -150,7 +152,7 @@ def cli(ctx): @cli.group() @click.pass_context def repository(ctx): - """ Repository management commands """ + """ Repository management commands. """ pass @@ -158,7 +160,7 @@ def repository(ctx): @cli.group() @click.pass_context def show(ctx): - """ Package manager show commands """ + """ Package manager show commands. """ pass @@ -166,7 +168,7 @@ def show(ctx): @show.group() @click.pass_context def package(ctx): - """ Package show commands """ + """ Package show commands. """ pass @@ -174,7 +176,7 @@ def package(ctx): @cli.command() @click.pass_context def list(ctx): - """ List available packages """ + """ List available packages. """ table_header = ['Name', 'Repository', 'Description', 'Version', 'Status'] table_body = [] @@ -208,7 +210,7 @@ def manifest(ctx, package_expr, from_repository, from_tarball): - """ Show package manifest """ + """ Show package manifest. """ manager: PackageManager = ctx.obj @@ -224,11 +226,11 @@ def manifest(ctx, @package.command() @click.argument('name') -@click.option('--all', is_flag=True, help='Show all available tags in repository') -@click.option('--plain', is_flag=True, help='Plain output') +@click.option('--all', is_flag=True, help='Show all available tags in repository.') +@click.option('--plain', is_flag=True, help='Plain output.') @click.pass_context def versions(ctx, name, all, plain): - """ Show available versions """ + """ Show available versions. """ try: manager: PackageManager = ctx.obj @@ -248,7 +250,7 @@ def changelog(ctx, package_expr, from_repository, from_tarball): - """ Show package changelog """ + """ Show package changelog. """ manager: PackageManager = ctx.obj @@ -281,12 +283,12 @@ def changelog(ctx, @repository.command() @click.argument('name', type=str) @click.argument('repository', type=str) -@click.option('--default-reference', type=str, help='Default installation reference') -@click.option('--description', type=str, help='Optional package entry description') +@click.option('--default-reference', type=str, help='Default installation reference. Can be a tag or sha256 digest in repository.') +@click.option('--description', type=str, help='Optional package entry description.') @click.pass_context @root_privileges_required def add(ctx, name, repository, default_reference, description): - """ Add a new repository to database """ + """ Add a new repository to database. """ manager: PackageManager = ctx.obj @@ -320,11 +322,16 @@ def remove(ctx, name): help='Set the default state of the feature to enabled ' 'and enable feature right after installation. ' 'NOTE: user needs to execute "config save -y" to make ' - 'this setting persistent') -@click.option('--default-owner', + 'this setting persistent.') +@click.option('--set-owner', type=click.Choice(['local', 'kube']), default='local', - help='Default owner configuration setting for a feature') + help='Default owner configuration setting for a feature.') +@click.option('--allow-downgrade', + is_flag=True, + help='Allow package downgrade. By default an attempt to downgrade the package ' + 'will result in a failure since downgrade might not be supported by the package, ' + 'thus requires explicit request from the user.') @add_options(PACKAGE_SOURCE_OPTIONS) @add_options(PACKAGE_COMMON_OPERATION_OPTIONS) @add_options(PACKAGE_COMMON_INSTALL_OPTIONS) @@ -337,10 +344,16 @@ def install(ctx, force, yes, enable, - default_owner, - skip_host_plugins): - """ Install/Upgrade package using [PACKAGE_EXPR] in format - "[=|@]" """ + set_owner, + skip_host_plugins, + allow_downgrade): + """ Install/Upgrade package using [PACKAGE_EXPR] in format "[=|@]". + + The repository to pull the package from is resolved by lookup in package database, + thus the package has to be added via "sonic-package-manager repository add" command. + + In case when [PACKAGE_EXPR] is a package name "" this command will install or upgrade + to a version referenced by "default-reference" in package database. """ manager: PackageManager = ctx.obj @@ -355,8 +368,9 @@ def install(ctx, install_opts = { 'force': force, 'enable': enable, - 'default_owner': default_owner, + 'default_owner': set_owner, 'skip_host_plugins': skip_host_plugins, + 'allow_downgrade': allow_downgrade, } try: @@ -377,7 +391,7 @@ def install(ctx, @click.pass_context @root_privileges_required def reset(ctx, name, force, yes, skip_host_plugins): - """ Reset package to the default version """ + """ Reset package to the default version. """ manager: PackageManager = ctx.obj @@ -399,7 +413,7 @@ def reset(ctx, name, force, yes, skip_host_plugins): @click.pass_context @root_privileges_required def uninstall(ctx, name, force, yes): - """ Uninstall package """ + """ Uninstall package. """ manager: PackageManager = ctx.obj @@ -422,7 +436,7 @@ def uninstall(ctx, name, force, yes): @click.pass_context @root_privileges_required def migrate(ctx, database, force, yes, dockerd_socket): - """ Migrate packages from the given database file """ + """ Migrate packages from the given database file. """ manager: PackageManager = ctx.obj diff --git a/sonic_package_manager/manager.py b/sonic_package_manager/manager.py index 527869ac73..fb15c91a7c 100644 --- a/sonic_package_manager/manager.py +++ b/sonic_package_manager/manager.py @@ -91,11 +91,7 @@ def opt_check(func: Callable) -> Callable: @functools.wraps(func) def wrapped_function(*args, **kwargs): sig = signature(func) - redundant_opts = [opt for opt in kwargs if opt not in sig.parameters] - if redundant_opts: - raise PackageManagerError( - f'Unsupported options: {",".join(redundant_opts)} for {func.__name__}' - ) + kwargs = {k: v for k, v in kwargs.items() if k in sig.parameters} return func(*args, **kwargs) return wrapped_function From 84e0b586a0631e169606067ec1c2afd630e335f9 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Mon, 26 Apr 2021 12:55:50 +0300 Subject: [PATCH 056/173] yang Signed-off-by: Stepan Blyschak --- config/config_mgmt.py | 11 +- sonic_package_manager/manager.py | 4 +- .../service_creator/creator.py | 22 +- .../service_creator/sonic_db.py | 2 + tests/sonic_package_manager/conftest.py | 234 +++++++++++++++++- .../test_service_creator.py | 4 +- 6 files changed, 256 insertions(+), 21 deletions(-) diff --git a/config/config_mgmt.py b/config/config_mgmt.py index 45f18b32c1..5f35127803 100644 --- a/config/config_mgmt.py +++ b/config/config_mgmt.py @@ -6,6 +6,7 @@ import os import re import syslog +import yang as ly from json import load from sys import flags from time import sleep as tsleep @@ -234,9 +235,9 @@ def add_module(self, yang_module_text, replace_if_exists=False): module_path = os.path.join(YANG_DIR, '{}.yang'.format(module_name)) if os.path.exists(module_path) and not replace_if_exists: raise Exception('{} already exists'.format(module_name)) + with open(module_path, 'w') as module_file: + module_file.write(yang_module_text) try: - with open(module_path, 'w') as module_file: - module_file.write(yang_module_text) self.__init_sonic_yang() except Exception: os.remove(module_path) @@ -254,9 +255,11 @@ def remove_module(self, module_name): """ module_path = os.path.join(YANG_DIR, '{}.yang'.format(module_name)) + if not os.path.exists(module_path): + return + with open(module_path, 'r') as module_file: + yang_module_text = module_file.read() try: - with open(module_path, 'r') as module_file: - yang_module_text = module_file.read() os.remove(module_path) self.__init_sonic_yang() except Exception: diff --git a/sonic_package_manager/manager.py b/sonic_package_manager/manager.py index 452898deca..976d65d092 100644 --- a/sonic_package_manager/manager.py +++ b/sonic_package_manager/manager.py @@ -253,8 +253,7 @@ def __init__(self, metadata_resolver: MetadataResolver, service_creator: ServiceCreator, device_information: Any, - lock: filelock.FileLock, - cfg_mgmt: config_mgmt.ConfigMgmt): + lock: filelock.FileLock): """ Initialize PackageManager. """ self.lock = lock @@ -267,7 +266,6 @@ def __init__(self, self.is_multi_npu = device_information.is_multi_npu() self.num_npus = device_information.get_num_npus() self.version_info = device_information.get_sonic_version_info() - self.cfg_mgmt = cfg_mgmt @under_lock def add_repository(self, *args, **kwargs): diff --git a/sonic_package_manager/service_creator/creator.py b/sonic_package_manager/service_creator/creator.py index 6bea5b14c9..b9cc22b008 100644 --- a/sonic_package_manager/service_creator/creator.py +++ b/sonic_package_manager/service_creator/creator.py @@ -148,18 +148,19 @@ def remove_file(path): remove_file(os.path.join(SERVICE_MGMT_SCRIPT_LOCATION, f'{name}.sh')) remove_file(os.path.join(DOCKER_CTL_SCRIPT_LOCATION, f'{name}.sh')) remove_file(os.path.join(DEBUG_DUMP_SCRIPT_LOCATION, f'{name}')) - self.uninstall_yang_module(package) self.update_dependent_list_file(package, remove=True) + if deregister_feature and not keep_config: + self.remove_config(package) + + self.uninstall_yang_module(package) + self.post_operation_hook() if deregister_feature: self.feature_registry.deregister(package.manifest['service']['name']) - if deregister_feature and not keep_config: - self.remove_config(package) - def post_operation_hook(self): if not in_chroot(): run_command('systemctl daemon-reload') @@ -336,10 +337,14 @@ def set_initial_config(self, package): cfg.update(old_fvs) fvs = list(cfg.items()) table.set(key, fvs) - - self.validate_configs() + + if package.metadata.yang_module_text: + self.validate_configs() def remove_config(self, package): + if not package.metadata.yang_module_text: + return + module_name = self.cfg_mgmt.get_module_name(package.metadata.yang_module_text) for tablename, module in self.cfg_mgmt.sy.confDbYangMap.items(): if module['module'] != module_name: @@ -366,8 +371,13 @@ def validate_configs(self): self.cfg_mgmt.loadData(self.cfg_mgmt.configdbJsonIn) def install_yang_module(self, package: Package): + if not package.metadata.yang_module_text: + return + self.cfg_mgmt.add_module(package.metadata.yang_module_text) def uninstall_yang_module(self, package: Package): + if not package.metadata.yang_module_text: + return module_name = self.cfg_mgmt.get_module_name(package.metadata.yang_module_text) self.cfg_mgmt.remove_module(module_name) diff --git a/sonic_package_manager/service_creator/sonic_db.py b/sonic_package_manager/service_creator/sonic_db.py index 8f3be792cc..07a946f33e 100644 --- a/sonic_package_manager/service_creator/sonic_db.py +++ b/sonic_package_manager/service_creator/sonic_db.py @@ -51,6 +51,8 @@ def _del(self, key): with contextlib.suppress(KeyError): config[self._table].pop(key) + if not config[self._table]: + config.pop(self._table) with open(self._file, 'w') as stream: json.dump(config, stream, indent=4) diff --git a/tests/sonic_package_manager/conftest.py b/tests/sonic_package_manager/conftest.py index e3a3f5d458..b18a01acb8 100644 --- a/tests/sonic_package_manager/conftest.py +++ b/tests/sonic_package_manager/conftest.py @@ -1,5 +1,7 @@ #!/usr/bin/env python +import json + from dataclasses import dataclass from unittest import mock from unittest.mock import Mock, MagicMock @@ -7,6 +9,8 @@ import pytest from docker_image.reference import Reference +from config.config_mgmt import ConfigMgmt + from sonic_package_manager.database import PackageDatabase, PackageEntry from sonic_package_manager.manager import DockerApi, PackageManager from sonic_package_manager.manifest import Manifest @@ -16,6 +20,211 @@ from sonic_package_manager.service_creator.creator import * +CONFIG_DB_JSON_DATA = { + "ACL_TABLE": { + "NO-NSW-PACL-TEST": { + "policy_desc": "NO-NSW-PACL-TEST", + "type": "L3", + "stage": "INGRESS", + "ports": [ + "Ethernet9", + "Ethernet11", + ] + }, + "NO-NSW-PACL-V4": { + "policy_desc": "NO-NSW-PACL-V4", + "type": "L3", + "stage": "INGRESS", + "ports": [ + "Ethernet0", + "Ethernet4", + "Ethernet8", + "Ethernet10" + ] + } + }, + "VLAN": { + "Vlan100": { + "admin_status": "up", + "description": "server_vlan", + "dhcp_servers": [ + "10.186.72.116" + ] + }, + }, + "VLAN_MEMBER": { + "Vlan100|Ethernet0": { + "tagging_mode": "untagged" + }, + "Vlan100|Ethernet2": { + "tagging_mode": "untagged" + }, + "Vlan100|Ethernet8": { + "tagging_mode": "untagged" + }, + "Vlan100|Ethernet11": { + "tagging_mode": "untagged" + }, + }, + "INTERFACE": { + "Ethernet10": {}, + "Ethernet10|2a04:0000:40:a709::1/126": { + "scope": "global", + "family": "IPv6" + } + }, + "PORT": { + "Ethernet0": { + "alias": "Eth1/1", + "lanes": "65", + "description": "", + "speed": "25000", + "admin_status": "up" + }, + "Ethernet1": { + "alias": "Eth1/2", + "lanes": "66", + "description": "", + "speed": "25000", + "admin_status": "up" + }, + "Ethernet2": { + "alias": "Eth1/3", + "lanes": "67", + "description": "", + "speed": "25000", + "admin_status": "up" + }, + "Ethernet3": { + "alias": "Eth1/4", + "lanes": "68", + "description": "", + "speed": "25000", + "admin_status": "up" + }, + "Ethernet4": { + "alias": "Eth2/1", + "lanes": "69", + "description": "", + "speed": "25000", + "admin_status": "up" + }, + "Ethernet5": { + "alias": "Eth2/2", + "lanes": "70", + "description": "", + "speed": "25000", + "admin_status": "up" + }, + "Ethernet6": { + "alias": "Eth2/3", + "lanes": "71", + "description": "", + "speed": "25000", + "admin_status": "up" + }, + "Ethernet7": { + "alias": "Eth2/4", + "lanes": "72", + "description": "", + "speed": "25000", + "admin_status": "up" + }, + "Ethernet8": { + "alias": "Eth3/1", + "lanes": "73", + "description": "", + "speed": "25000", + "admin_status": "up" + }, + "Ethernet9": { + "alias": "Eth3/2", + "lanes": "74", + "description": "", + "speed": "25000", + "admin_status": "up" + }, + "Ethernet10": { + "alias": "Eth3/3", + "lanes": "75", + "description": "", + "speed": "25000", + "admin_status": "up" + }, + "Ethernet11": { + "alias": "Eth3/4", + "lanes": "76", + "description": "", + "speed": "25000", + "admin_status": "up" + } + } +} + + +TEST_YANG = """ +module sonic-test { + + yang-version 1.1; + + namespace "http://github.com/Azure/sonic-test"; + prefix test; + + description "Test yang Module for SONiC OS"; + + revision 2020-05-01 { + description "First Revision"; + } + + container sonic-test { + + container TEST_OBJ { + + description "Test objects configuration"; + + list TEST_OBJ_LIST { + + description "List of Test objects"; + + key "name"; + + leaf name { + type string { + pattern "l1|forwarding|buffer"; + } + } + + leaf "type" { + type string { + pattern "type1|type2"; + } + } + } + /* end of TEST_OBJ_LIST */ + } + /* end of TEST_OBJ container */ + + container TEST { + + description "Test global configuration"; + + container global { + + leaf mode { + type string { + pattern "debug"; + } + default "debug"; + } + } + } + /* end of container TEST */ + } + /* end of container sonic-test */ + } +/* end of module sonic-test */ +""" + @pytest.fixture def mock_docker_api(): docker = MagicMock(DockerApi) @@ -67,7 +276,14 @@ def mock_sonic_db(): @pytest.fixture def mock_config_mgmt(): - yield MagicMock() + with open('config_db.json', 'w') as cfg: + json.dump(CONFIG_DB_JSON_DATA, cfg) + yield ConfigMgmt(source='config_db.json') + + +@pytest.fixture +def test_yang(): + yield TEST_YANG @pytest.fixture @@ -80,7 +296,8 @@ def __init__(self): components={ 'libswsscommon': Version.parse('1.0.0'), 'libsairedis': Version.parse('1.0.0') - } + }, + yang=TEST_YANG, ) self.add('Azure/docker-test', '1.6.0', 'test-package', '1.6.0') self.add('Azure/docker-test-2', '1.5.0', 'test-package-2', '1.5.0') @@ -99,21 +316,24 @@ def __init__(self): def from_registry(self, repository: str, reference: str): manifest = Manifest.marshal(self.metadata_store[repository][reference]['manifest']) components = self.metadata_store[repository][reference]['components'] - return Metadata(manifest, components) + yang = self.metadata_store[repository][reference]['yang'] + return Metadata(manifest, components, yang) def from_local(self, image: str): ref = Reference.parse(image) manifest = Manifest.marshal(self.metadata_store[ref['name']][ref['tag']]['manifest']) components = self.metadata_store[ref['name']][ref['tag']]['components'] - return Metadata(manifest, components) + yang = self.metadata_store[ref['name']][ref['tag']]['yang'] + return Metadata(manifest, components, yang) def from_tarball(self, filepath: str) -> Manifest: path, ref = filepath.split(':') manifest = Manifest.marshal(self.metadata_store[path][ref]['manifest']) components = self.metadata_store[path][ref]['components'] - return Metadata(manifest, components) + yang = self.metadata_store[path][ref]['yang'] + return Metadata(manifest, components, yang) - def add(self, repo, reference, name, version, components=None): + def add(self, repo, reference, name, version, components=None, yang=None): repo_dict = self.metadata_store.setdefault(repo, {}) repo_dict[reference] = { 'manifest': { @@ -127,6 +347,7 @@ def add(self, repo, reference, name, version, components=None): } }, 'components': components or {}, + 'yang': yang, } yield FakeMetadataResolver() @@ -338,6 +559,7 @@ def fake_db_for_migration(fake_metadata_resolver): @pytest.fixture() def sonic_fs(fs): fs.create_file('/proc/1/root') + fs.create_dir('/usr/local/yang-models') fs.create_dir(ETC_SONIC_PATH) fs.create_dir(SYSTEMD_LOCATION) fs.create_dir(DOCKER_CTL_SCRIPT_LOCATION) diff --git a/tests/sonic_package_manager/test_service_creator.py b/tests/sonic_package_manager/test_service_creator.py index da2fa26e49..d16e6824b2 100644 --- a/tests/sonic_package_manager/test_service_creator.py +++ b/tests/sonic_package_manager/test_service_creator.py @@ -83,7 +83,7 @@ def test_service_creator_with_debug_dump(sonic_fs, manifest, mock_feature_regist def test_service_creator_initial_config(sonic_fs, manifest, mock_feature_registry, - mock_sonic_db, mock_config_mgmt): + mock_sonic_db, mock_config_mgmt, test_yang): mock_table = Mock() mock_table.get = Mock(return_value=(True, (('field_2', 'original_value_2'),))) mock_sonic_db.initial_table = Mock(return_value=mock_table) @@ -93,7 +93,7 @@ def test_service_creator_initial_config(sonic_fs, manifest, mock_feature_registr creator = ServiceCreator(mock_feature_registry, mock_sonic_db, mock_config_mgmt) entry = PackageEntry('test', 'azure/sonic-test') - package = Package(entry, Metadata(manifest)) + package = Package(entry, Metadata(manifest, yang_module_text=test_yang)) creator.create(package) assert not sonic_fs.exists(os.path.join(DEBUG_DUMP_SCRIPT_LOCATION, 'test')) From 1ace0e121b8b2367273d700056ecbe2349e6dfed Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Mon, 26 Apr 2021 13:11:09 +0300 Subject: [PATCH 057/173] fix command line options for install/upgrade Signed-off-by: Stepan Blyschak --- sonic_package_manager/main.py | 13 +++++++++---- sonic_package_manager/manager.py | 6 +++++- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/sonic_package_manager/main.py b/sonic_package_manager/main.py index 70e1e67310..c0589ae5b5 100644 --- a/sonic_package_manager/main.py +++ b/sonic_package_manager/main.py @@ -319,16 +319,18 @@ def remove(ctx, name): @cli.command() @click.option('--enable', is_flag=True, + default=None, help='Set the default state of the feature to enabled ' 'and enable feature right after installation. ' 'NOTE: user needs to execute "config save -y" to make ' 'this setting persistent.') @click.option('--set-owner', type=click.Choice(['local', 'kube']), - default='local', + default=None, help='Default owner configuration setting for a feature.') @click.option('--allow-downgrade', is_flag=True, + default=None, help='Allow package downgrade. By default an attempt to downgrade the package ' 'will result in a failure since downgrade might not be supported by the package, ' 'thus requires explicit request from the user.') @@ -367,11 +369,14 @@ def install(ctx, install_opts = { 'force': force, - 'enable': enable, - 'default_owner': set_owner, 'skip_host_plugins': skip_host_plugins, - 'allow_downgrade': allow_downgrade, } + if enable is not None: + install_opts['enable'] = enable + if set_owner is not None: + install_opts['default_owner'] = set_owner + if allow_downgrade is not None: + install_opts['allow_downgrade'] = allow_downgrade try: manager.install(package_expr, diff --git a/sonic_package_manager/manager.py b/sonic_package_manager/manager.py index fb15c91a7c..9f3f5d95d4 100644 --- a/sonic_package_manager/manager.py +++ b/sonic_package_manager/manager.py @@ -91,7 +91,11 @@ def opt_check(func: Callable) -> Callable: @functools.wraps(func) def wrapped_function(*args, **kwargs): sig = signature(func) - kwargs = {k: v for k, v in kwargs.items() if k in sig.parameters} + unsupported_opts = [opt for opt in kwargs if opt not in sig.parameters] + if unsupported_opts: + raise PackageManagerError( + f'Unsupported options {unsupported_opts} for {func.__name__}' + ) return func(*args, **kwargs) return wrapped_function From 5133947ef671656584f4b49340b644d62e8c72fd Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Mon, 26 Apr 2021 14:09:14 +0300 Subject: [PATCH 058/173] yang Signed-off-by: Stepan Blyschak --- sonic_package_manager/manager.py | 20 +- tests/sonic_package_manager/conftest.py | 219 +----------------- .../test_service_creator.py | 19 +- 3 files changed, 27 insertions(+), 231 deletions(-) diff --git a/sonic_package_manager/manager.py b/sonic_package_manager/manager.py index a94a7a2541..c1e6e639d8 100644 --- a/sonic_package_manager/manager.py +++ b/sonic_package_manager/manager.py @@ -594,16 +594,16 @@ def migrate_packages(self, old_package_database: PackageDatabase, dockerd_sock: Optional[str] = None): """ - Migrate packages from old database. This function can do a comparison between - current database and the database passed in as argument. If the package is - missing in the current database it will be added. If the package is installed - in the passed database and in the current it is not installed it will be - installed with a passed database package version. If the package is installed - in the passed database and it is installed in the current database but with - older version the package will be upgraded to the never version. If the package - is installed in the passed database and in the current it is installed but with - never version - no actions are taken. If dockerd_sock parameter is passed, the - migration process will use loaded images from docker library of the currently + Migrate packages from old database. This function can do a comparison between + current database and the database passed in as argument. If the package is + missing in the current database it will be added. If the package is installed + in the passed database and in the current it is not installed it will be + installed with a passed database package version. If the package is installed + in the passed database and it is installed in the current database but with + older version the package will be upgraded to the never version. If the package + is installed in the passed database and in the current it is installed but with + never version - no actions are taken. If dockerd_sock parameter is passed, the + migration process will use loaded images from docker library of the currently installed image. Args: diff --git a/tests/sonic_package_manager/conftest.py b/tests/sonic_package_manager/conftest.py index b18a01acb8..c4d50672be 100644 --- a/tests/sonic_package_manager/conftest.py +++ b/tests/sonic_package_manager/conftest.py @@ -20,211 +20,6 @@ from sonic_package_manager.service_creator.creator import * -CONFIG_DB_JSON_DATA = { - "ACL_TABLE": { - "NO-NSW-PACL-TEST": { - "policy_desc": "NO-NSW-PACL-TEST", - "type": "L3", - "stage": "INGRESS", - "ports": [ - "Ethernet9", - "Ethernet11", - ] - }, - "NO-NSW-PACL-V4": { - "policy_desc": "NO-NSW-PACL-V4", - "type": "L3", - "stage": "INGRESS", - "ports": [ - "Ethernet0", - "Ethernet4", - "Ethernet8", - "Ethernet10" - ] - } - }, - "VLAN": { - "Vlan100": { - "admin_status": "up", - "description": "server_vlan", - "dhcp_servers": [ - "10.186.72.116" - ] - }, - }, - "VLAN_MEMBER": { - "Vlan100|Ethernet0": { - "tagging_mode": "untagged" - }, - "Vlan100|Ethernet2": { - "tagging_mode": "untagged" - }, - "Vlan100|Ethernet8": { - "tagging_mode": "untagged" - }, - "Vlan100|Ethernet11": { - "tagging_mode": "untagged" - }, - }, - "INTERFACE": { - "Ethernet10": {}, - "Ethernet10|2a04:0000:40:a709::1/126": { - "scope": "global", - "family": "IPv6" - } - }, - "PORT": { - "Ethernet0": { - "alias": "Eth1/1", - "lanes": "65", - "description": "", - "speed": "25000", - "admin_status": "up" - }, - "Ethernet1": { - "alias": "Eth1/2", - "lanes": "66", - "description": "", - "speed": "25000", - "admin_status": "up" - }, - "Ethernet2": { - "alias": "Eth1/3", - "lanes": "67", - "description": "", - "speed": "25000", - "admin_status": "up" - }, - "Ethernet3": { - "alias": "Eth1/4", - "lanes": "68", - "description": "", - "speed": "25000", - "admin_status": "up" - }, - "Ethernet4": { - "alias": "Eth2/1", - "lanes": "69", - "description": "", - "speed": "25000", - "admin_status": "up" - }, - "Ethernet5": { - "alias": "Eth2/2", - "lanes": "70", - "description": "", - "speed": "25000", - "admin_status": "up" - }, - "Ethernet6": { - "alias": "Eth2/3", - "lanes": "71", - "description": "", - "speed": "25000", - "admin_status": "up" - }, - "Ethernet7": { - "alias": "Eth2/4", - "lanes": "72", - "description": "", - "speed": "25000", - "admin_status": "up" - }, - "Ethernet8": { - "alias": "Eth3/1", - "lanes": "73", - "description": "", - "speed": "25000", - "admin_status": "up" - }, - "Ethernet9": { - "alias": "Eth3/2", - "lanes": "74", - "description": "", - "speed": "25000", - "admin_status": "up" - }, - "Ethernet10": { - "alias": "Eth3/3", - "lanes": "75", - "description": "", - "speed": "25000", - "admin_status": "up" - }, - "Ethernet11": { - "alias": "Eth3/4", - "lanes": "76", - "description": "", - "speed": "25000", - "admin_status": "up" - } - } -} - - -TEST_YANG = """ -module sonic-test { - - yang-version 1.1; - - namespace "http://github.com/Azure/sonic-test"; - prefix test; - - description "Test yang Module for SONiC OS"; - - revision 2020-05-01 { - description "First Revision"; - } - - container sonic-test { - - container TEST_OBJ { - - description "Test objects configuration"; - - list TEST_OBJ_LIST { - - description "List of Test objects"; - - key "name"; - - leaf name { - type string { - pattern "l1|forwarding|buffer"; - } - } - - leaf "type" { - type string { - pattern "type1|type2"; - } - } - } - /* end of TEST_OBJ_LIST */ - } - /* end of TEST_OBJ container */ - - container TEST { - - description "Test global configuration"; - - container global { - - leaf mode { - type string { - pattern "debug"; - } - default "debug"; - } - } - } - /* end of container TEST */ - } - /* end of container sonic-test */ - } -/* end of module sonic-test */ -""" - @pytest.fixture def mock_docker_api(): docker = MagicMock(DockerApi) @@ -276,14 +71,7 @@ def mock_sonic_db(): @pytest.fixture def mock_config_mgmt(): - with open('config_db.json', 'w') as cfg: - json.dump(CONFIG_DB_JSON_DATA, cfg) - yield ConfigMgmt(source='config_db.json') - - -@pytest.fixture -def test_yang(): - yield TEST_YANG + yield MagicMock() @pytest.fixture @@ -297,7 +85,6 @@ def __init__(self): 'libswsscommon': Version.parse('1.0.0'), 'libsairedis': Version.parse('1.0.0') }, - yang=TEST_YANG, ) self.add('Azure/docker-test', '1.6.0', 'test-package', '1.6.0') self.add('Azure/docker-test-2', '1.5.0', 'test-package-2', '1.5.0') @@ -559,7 +346,6 @@ def fake_db_for_migration(fake_metadata_resolver): @pytest.fixture() def sonic_fs(fs): fs.create_file('/proc/1/root') - fs.create_dir('/usr/local/yang-models') fs.create_dir(ETC_SONIC_PATH) fs.create_dir(SYSTEMD_LOCATION) fs.create_dir(DOCKER_CTL_SCRIPT_LOCATION) @@ -590,8 +376,7 @@ def package_manager(mock_docker_api, fake_db, fake_metadata_resolver, mock_service_creator, fake_device_info, - MagicMock(), - mock_config_mgmt) + MagicMock()) @pytest.fixture diff --git a/tests/sonic_package_manager/test_service_creator.py b/tests/sonic_package_manager/test_service_creator.py index d16e6824b2..d34ac0070f 100644 --- a/tests/sonic_package_manager/test_service_creator.py +++ b/tests/sonic_package_manager/test_service_creator.py @@ -82,9 +82,10 @@ def test_service_creator_with_debug_dump(sonic_fs, manifest, mock_feature_regist assert sonic_fs.exists(os.path.join(DEBUG_DUMP_SCRIPT_LOCATION, 'test')) -def test_service_creator_initial_config(sonic_fs, manifest, mock_feature_registry, - mock_sonic_db, mock_config_mgmt, test_yang): +def test_service_creator_yang(sonic_fs, manifest, mock_feature_registry, + mock_sonic_db, mock_config_mgmt): mock_table = Mock() + test_yang = 'TEST YANG' mock_table.get = Mock(return_value=(True, (('field_2', 'original_value_2'),))) mock_sonic_db.initial_table = Mock(return_value=mock_table) mock_sonic_db.persistent_table = Mock(return_value=mock_table) @@ -96,7 +97,8 @@ def test_service_creator_initial_config(sonic_fs, manifest, mock_feature_registr package = Package(entry, Metadata(manifest, yang_module_text=test_yang)) creator.create(package) - assert not sonic_fs.exists(os.path.join(DEBUG_DUMP_SCRIPT_LOCATION, 'test')) + mock_config_mgmt.add_module.assert_called_with('TEST YANG') + mock_config_mgmt.get_module_name = Mock(return_value='sonic-test') manifest['package']['init-cfg'] = { 'TABLE_A': { @@ -106,14 +108,23 @@ def test_service_creator_initial_config(sonic_fs, manifest, mock_feature_registr }, }, } - package = Package(entry, Metadata(manifest)) + package = Package(entry, Metadata(manifest, yang_module_text=test_yang)) creator.create(package) + + mock_config_mgmt.add_module.assert_called_with('TEST YANG') + mock_table.set.assert_called_with('key_a', [('field_1', 'value_1'), ('field_2', 'original_value_2')]) + mock_config_mgmt.sy.confDbYangMap = { + 'TABLE_A': {'module': 'sonic-test'} + } + mock_table.getKeys = Mock(return_value=['key_a']) + creator.remove(package) mock_table._del.assert_called_with('key_a') + mock_config_mgmt.remove_module.assert_called_with('sonic-test') def test_feature_registration(mock_sonic_db, manifest): From 0fc64639678c1001937a0828800f86bb0b9d26c6 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Mon, 26 Apr 2021 17:15:50 +0300 Subject: [PATCH 059/173] fix bug when have components constraints Signed-off-by: Stepan Blyschak --- sonic_package_manager/constraint.py | 2 +- .../service_creator/creator.py | 2 +- .../sonic-what-just-happened.yang | 79 +++++++++++++++++++ 3 files changed, 81 insertions(+), 2 deletions(-) create mode 100644 sonic_package_manager/sonic-what-just-happened.yang diff --git a/sonic_package_manager/constraint.py b/sonic_package_manager/constraint.py index 8b044f3ec9..af5a13000b 100644 --- a/sonic_package_manager/constraint.py +++ b/sonic_package_manager/constraint.py @@ -58,7 +58,7 @@ def deparse(self) -> Dict[str, str]: """ return { - component: str(version) for component, version in self.components + component: str(version) for component, version in self.components.items() } diff --git a/sonic_package_manager/service_creator/creator.py b/sonic_package_manager/service_creator/creator.py index d86b70f979..2b305b04b3 100644 --- a/sonic_package_manager/service_creator/creator.py +++ b/sonic_package_manager/service_creator/creator.py @@ -123,7 +123,7 @@ def create(self, self.feature_registry.register(package.manifest, state, owner) except (Exception, KeyboardInterrupt): - self.remove(package, not register_feature) + self.remove(package, register_feature) raise def remove(self, package: Package, deregister_feature=True): diff --git a/sonic_package_manager/sonic-what-just-happened.yang b/sonic_package_manager/sonic-what-just-happened.yang new file mode 100644 index 0000000000..1d02926b6f --- /dev/null +++ b/sonic_package_manager/sonic-what-just-happened.yang @@ -0,0 +1,79 @@ +module sonic-what-just-happened { + + yang-version 1.1; + + namespace "http://github.com/Azure/sonic-what-just-happened"; + prefix what-just-happened; + + description "What-Just-Happened yang Module for SONiC OS"; + + revision 2020-05-01 { + description "First Revision"; + } + + container sonic-what-just-happened { + + container WJH_CHANNEL { + + description "What-Just-Happened channels configuration"; + + list WJH_CHANNEL_LIST { + + description "List of What-Just-Happened channels"; + + key "name"; + + leaf name { + type string { + pattern "l1|forwarding|buffer"; + } + } + + leaf "type" { + type string { + pattern "raw|aggregated|raw_and_aggregated"; + } + } + + leaf drop_category_list { + type string; + } + } + /* end of WJH_CHANNEL_LIST */ + } + /* end of WJH_CHANNEL container */ + + container WJH { + + description "What-Just-Happened global configuration"; + + container global { + + leaf mode { + type string { + pattern "debug"; + } + default "debug"; + } + + leaf nice_level { + type int { + range -20..19; + } + default 1; + } + + leaf pci_bandwidth { + type int { + range 0..100; + } + default 50; + } + + } + } + /* end of container WJH */ + } + /* end of container sonic-what-just-happened */ +} +/* end of module sonic-what-just-happened */ \ No newline at end of file From 6feda3645d9720d0c5a3499b02b97e8c73720591 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Mon, 26 Apr 2021 17:15:50 +0300 Subject: [PATCH 060/173] fix bug when have components constraints Signed-off-by: Stepan Blyschak --- sonic_package_manager/constraint.py | 2 +- sonic_package_manager/service_creator/creator.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sonic_package_manager/constraint.py b/sonic_package_manager/constraint.py index 8b044f3ec9..af5a13000b 100644 --- a/sonic_package_manager/constraint.py +++ b/sonic_package_manager/constraint.py @@ -58,7 +58,7 @@ def deparse(self) -> Dict[str, str]: """ return { - component: str(version) for component, version in self.components + component: str(version) for component, version in self.components.items() } diff --git a/sonic_package_manager/service_creator/creator.py b/sonic_package_manager/service_creator/creator.py index d86b70f979..2b305b04b3 100644 --- a/sonic_package_manager/service_creator/creator.py +++ b/sonic_package_manager/service_creator/creator.py @@ -123,7 +123,7 @@ def create(self, self.feature_registry.register(package.manifest, state, owner) except (Exception, KeyboardInterrupt): - self.remove(package, not register_feature) + self.remove(package, register_feature) raise def remove(self, package: Package, deregister_feature=True): From 548896a38dd083b29c23d0522f12b950b4c7677d Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Thu, 22 Apr 2021 10:52:55 +0000 Subject: [PATCH 061/173] Worked version Signed-off-by: Vadym Hlushko --- sonic_cli_gen/__init__.py | 0 sonic_cli_gen/generator.py | 31 +++++++++++++++++++++++++++++++ sonic_cli_gen/main.py | 27 +++++++++++++++++++++++++++ sonic_cli_gen/yang_parser.py | 36 ++++++++++++++++++++++++++++++++++++ 4 files changed, 94 insertions(+) create mode 100644 sonic_cli_gen/__init__.py create mode 100644 sonic_cli_gen/generator.py create mode 100644 sonic_cli_gen/main.py create mode 100644 sonic_cli_gen/yang_parser.py diff --git a/sonic_cli_gen/__init__.py b/sonic_cli_gen/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/sonic_cli_gen/generator.py b/sonic_cli_gen/generator.py new file mode 100644 index 0000000000..400ba41a47 --- /dev/null +++ b/sonic_cli_gen/generator.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python + +try: + from sonic_cli_gen.yang_parser import YangParser +except ImportError as e: + raise ImportError("%s - required module not found" % str(e)) + +class CliGenerator: + """ SONiC CLI generator. This class provides public API + for sonic-cli-gen python library. It can generate config, + show, sonic-clear commands """ + def __init__(self, + yang_model): + """ Initialize PackageManager. """ + + self.yang_model_name = yang_model + + def generate_config_plugin(self): + parser = YangParser(self.yang_model_name) + parser.yang_to_dict() + pass + + #TODO + def generate_show_plugin(self): + print ("show") + pass + + # to be implemented in the next Phases + def generate_sonic_clear_plugin(self): + print ("sonic-clear") + pass diff --git a/sonic_cli_gen/main.py b/sonic_cli_gen/main.py new file mode 100644 index 0000000000..542ad91867 --- /dev/null +++ b/sonic_cli_gen/main.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python + +try: + import sys + import os + import click + from sonic_cli_gen.generator import CliGenerator +except ImportError as e: + raise ImportError("%s - required module not found" % str(e)) + +@click.group() +@click.pass_context +def cli(ctx): + """ SONiC CLI generator """ + print ("cli") + pass + +@cli.command() +@click.pass_context +def generate_config(ctx): + """ List available packages """ + gen = CliGenerator('sonic-vlan.yang') + gen.generate_config_plugin() + pass + +if __name__ == '__main__': + cli() \ No newline at end of file diff --git a/sonic_cli_gen/yang_parser.py b/sonic_cli_gen/yang_parser.py new file mode 100644 index 0000000000..50740b1f10 --- /dev/null +++ b/sonic_cli_gen/yang_parser.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python + +try: + import os + import sys + import glob + import yang as ly +except ImportError as e: + raise ImportError("%s - required module not found" % str(e)) + + +YANG_DIR = "/usr/local/yang-models/" + +class YangParser: + """ YANG model parser """ + def __init__(self, + yang_model): + self.yang_model = yang_model + self.ly_ctx = None + + try: + self.ly_ctx = ly.Context(YANG_DIR) + except Exception as e: + self.fail(e) + + def fail(self, e): + print(e) + raise e + + def yang_to_dict(self): + print ("YANG TO DICT") + data = { + 'yang_dir': YANG_DIR. + 'yang_files': glob + } + pass \ No newline at end of file From 3350f27807b6ff2dcd609a2b444bd2be2b865f0f Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Fri, 23 Apr 2021 13:55:06 +0000 Subject: [PATCH 062/173] added stub for function to determine static or dynamic YANG Signed-off-by: Vadym Hlushko --- setup.py | 2 ++ sonic_cli_gen/main.py | 2 +- sonic_cli_gen/yang_parser.py | 31 ++++++++++++++++++------------- 3 files changed, 21 insertions(+), 14 deletions(-) diff --git a/setup.py b/setup.py index cd706eb433..9aff1ac9c2 100644 --- a/setup.py +++ b/setup.py @@ -52,6 +52,7 @@ 'undebug', 'utilities_common', 'watchdogutil', + 'sonic_cli_gen', ], package_data={ 'show': ['aliases.ini'], @@ -153,6 +154,7 @@ 'sonic_installer = sonic_installer.main:sonic_installer', # Deprecated 'undebug = undebug.main:cli', 'watchdogutil = watchdogutil.main:watchdogutil', + 'sonic-cli-gen = sonic_cli_gen.main:cli', ] }, install_requires=[ diff --git a/sonic_cli_gen/main.py b/sonic_cli_gen/main.py index 542ad91867..a8b01a7034 100644 --- a/sonic_cli_gen/main.py +++ b/sonic_cli_gen/main.py @@ -19,7 +19,7 @@ def cli(ctx): @click.pass_context def generate_config(ctx): """ List available packages """ - gen = CliGenerator('sonic-vlan.yang') + gen = CliGenerator('sonic-vlan') gen.generate_config_plugin() pass diff --git a/sonic_cli_gen/yang_parser.py b/sonic_cli_gen/yang_parser.py index 50740b1f10..dfaec45e43 100644 --- a/sonic_cli_gen/yang_parser.py +++ b/sonic_cli_gen/yang_parser.py @@ -3,34 +3,39 @@ try: import os import sys - import glob - import yang as ly + from config.config_mgmt import ConfigMgmt except ImportError as e: raise ImportError("%s - required module not found" % str(e)) -YANG_DIR = "/usr/local/yang-models/" - class YangParser: """ YANG model parser """ def __init__(self, yang_model): self.yang_model = yang_model - self.ly_ctx = None + self.conf_mgmt = None try: - self.ly_ctx = ly.Context(YANG_DIR) + self.conf_mgmt = ConfigMgmt() except Exception as e: - self.fail(e) + raise Exception("Failed to load the {} class".format(str(e))) def fail(self, e): print(e) raise e def yang_to_dict(self): - print ("YANG TO DICT") - data = { - 'yang_dir': YANG_DIR. - 'yang_files': glob - } - pass \ No newline at end of file + yang_model_type = self._determine_yang_model_type() + + if (yang_model_type == 'static'): + print('static') + pass + else: + pass + + def _determine_yang_model_type(self): + cond = True + if cond: + return 'static' + else: + return 'dynamic' \ No newline at end of file From 1d1643ac298bbd515e88d0e44d4bdce9ee6bcfde Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Mon, 26 Apr 2021 09:26:09 +0000 Subject: [PATCH 063/173] _find_index_of_yang_model() Signed-off-by: Vadym Hlushko --- sonic_cli_gen/yang_parser.py | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/sonic_cli_gen/yang_parser.py b/sonic_cli_gen/yang_parser.py index dfaec45e43..ef58a8f0f6 100644 --- a/sonic_cli_gen/yang_parser.py +++ b/sonic_cli_gen/yang_parser.py @@ -3,23 +3,27 @@ try: import os import sys + import pprint from config.config_mgmt import ConfigMgmt except ImportError as e: raise ImportError("%s - required module not found" % str(e)) +# Config DB schema view +STATIC_TABLE = 'static' +LIST_TABLE = 'list' class YangParser: """ YANG model parser """ def __init__(self, - yang_model): - self.yang_model = yang_model + yang_model_name): + self.yang_model_name = yang_model_name self.conf_mgmt = None try: self.conf_mgmt = ConfigMgmt() except Exception as e: raise Exception("Failed to load the {} class".format(str(e))) - + def fail(self, e): print(e) raise e @@ -27,15 +31,16 @@ def fail(self, e): def yang_to_dict(self): yang_model_type = self._determine_yang_model_type() - if (yang_model_type == 'static'): - print('static') - pass - else: - pass - def _determine_yang_model_type(self): - cond = True - if cond: - return 'static' - else: - return 'dynamic' \ No newline at end of file + y_index = self._find_index_of_yang_model() + print("INDEX {}".format(y_index)) + + def _find_index_of_yang_model(self): + for i in range(len(self.conf_mgmt.sy.yJson)): + if (self.conf_mgmt.sy.yJson[i]['module']['@name'] == self.yang_model_name): + return i + + + + + \ No newline at end of file From cd31f0d307b28aa8a8548a1a55d1050b8a3a6de1 Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Mon, 26 Apr 2021 14:05:52 +0000 Subject: [PATCH 064/173] added function to init list's for module, top, tables Signed-off-by: Vadym Hlushko --- sonic_cli_gen/generator.py | 2 +- sonic_cli_gen/yang_parser.py | 29 +++++++++++++++++++++++------ 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/sonic_cli_gen/generator.py b/sonic_cli_gen/generator.py index 400ba41a47..154d82409b 100644 --- a/sonic_cli_gen/generator.py +++ b/sonic_cli_gen/generator.py @@ -17,7 +17,7 @@ def __init__(self, def generate_config_plugin(self): parser = YangParser(self.yang_model_name) - parser.yang_to_dict() + parser.parse_yang_model() pass #TODO diff --git a/sonic_cli_gen/yang_parser.py b/sonic_cli_gen/yang_parser.py index ef58a8f0f6..c211f9e948 100644 --- a/sonic_cli_gen/yang_parser.py +++ b/sonic_cli_gen/yang_parser.py @@ -18,6 +18,11 @@ def __init__(self, yang_model_name): self.yang_model_name = yang_model_name self.conf_mgmt = None + #index of yang model inside conf_mgmt.sy.yJson object + self.idx_yJson = None + self.y_module = None + self.y_top_level_container = None + self.y_tables = None try: self.conf_mgmt = ConfigMgmt() @@ -28,17 +33,29 @@ def fail(self, e): print(e) raise e - def yang_to_dict(self): - yang_model_type = self._determine_yang_model_type() + def parse_yang_model(self): + self._init_yang_module_and_containers() - def _determine_yang_model_type(self): - y_index = self._find_index_of_yang_model() - print("INDEX {}".format(y_index)) + def _determine_tables_type(self): + #for table in y_top_level_container['container']: + pass + def _init_yang_module_and_containers(self): + self._find_index_of_yang_model() + + self.y_module = self.conf_mgmt.sy.yJson[self.idx_yJson]['module'] + if self.y_module.get('container') is not None: + self.y_top_level_container = self.y_module['container'] + self.y_tables = self.y_top_level_container['container'] + import pdb; pdb.set_trace() + else: + raise KeyError('YANG model {} does NOT have "container" element'.format(self.yang_model_name)) + + # find index of yang_model inside yJson object def _find_index_of_yang_model(self): for i in range(len(self.conf_mgmt.sy.yJson)): if (self.conf_mgmt.sy.yJson[i]['module']['@name'] == self.yang_model_name): - return i + self.idx_yJson = i From 81b803de3f144105d46057a92a9bdb81596109dd Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Mon, 26 Apr 2021 15:32:05 +0000 Subject: [PATCH 065/173] Added func to determine - static or list, +comments Signed-off-by: Vadym Hlushko --- sonic_cli_gen/yang_parser.py | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/sonic_cli_gen/yang_parser.py b/sonic_cli_gen/yang_parser.py index c211f9e948..77f9ded716 100644 --- a/sonic_cli_gen/yang_parser.py +++ b/sonic_cli_gen/yang_parser.py @@ -4,25 +4,27 @@ import os import sys import pprint + from collections import OrderedDict from config.config_mgmt import ConfigMgmt except ImportError as e: raise ImportError("%s - required module not found" % str(e)) -# Config DB schema view -STATIC_TABLE = 'static' -LIST_TABLE = 'list' - class YangParser: """ YANG model parser """ def __init__(self, yang_model_name): self.yang_model_name = yang_model_name self.conf_mgmt = None - #index of yang model inside conf_mgmt.sy.yJson object + # index of yang model inside conf_mgmt.sy.yJson object self.idx_yJson = None - self.y_module = None - self.y_top_level_container = None - self.y_tables = None + # 'module' entity from .yang file + self.y_module = OrderedDict() + # top level 'container' entity from .yang file + self.y_top_level_container = OrderedDict() + # 'container' entities from .yang file + self.y_tables = list() + # dictionary that represent Config DB schema + self.yang_2_dict = OrderedDict() try: self.conf_mgmt = ConfigMgmt() @@ -36,9 +38,14 @@ def fail(self, e): def parse_yang_model(self): self._init_yang_module_and_containers() + self._determine_tables_type() + def _determine_tables_type(self): - #for table in y_top_level_container['container']: - pass + for table in self.y_tables: + if table.get('list') is None: + self.yang_2_dict[table.get('@name')] = {'type': 'static'} + else: + self.yang_2_dict[table.get('@name')] = {'type': 'list'} def _init_yang_module_and_containers(self): self._find_index_of_yang_model() @@ -47,7 +54,6 @@ def _init_yang_module_and_containers(self): if self.y_module.get('container') is not None: self.y_top_level_container = self.y_module['container'] self.y_tables = self.y_top_level_container['container'] - import pdb; pdb.set_trace() else: raise KeyError('YANG model {} does NOT have "container" element'.format(self.yang_model_name)) From ecdf8b6a58a9f1b44a3076d3a1a2a3a9300f8c54 Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Wed, 28 Apr 2021 10:07:08 +0000 Subject: [PATCH 066/173] Added auto compleation file Signed-off-by: Vadym Hlushko --- sonic-utilities-data/bash_completion.d/sonic-cli-gen | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 sonic-utilities-data/bash_completion.d/sonic-cli-gen diff --git a/sonic-utilities-data/bash_completion.d/sonic-cli-gen b/sonic-utilities-data/bash_completion.d/sonic-cli-gen new file mode 100644 index 0000000000..3327f9c513 --- /dev/null +++ b/sonic-utilities-data/bash_completion.d/sonic-cli-gen @@ -0,0 +1,8 @@ +_sonic_cli_gen_completion() { + COMPREPLY=( $( env COMP_WORDS="${COMP_WORDS[*]}" \ + COMP_CWORD=$COMP_CWORD \ + _SONIC_CLI_GEN_COMPLETE=complete $1 ) ) + return 0 +} + +complete -F _sonic_cli_gen_completion -o default sonic-cli-gen; From 42779c7d3ef336988483c64688b224dae6017fb2 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Wed, 28 Apr 2021 18:12:36 +0300 Subject: [PATCH 067/173] Add CLI Jinja templates Signed-off-by: Stepan Blyschak --- sonic-utilities-data/debian/install | 5 +- .../templates/sonic-cli-gen/common.j2 | 11 + .../templates/sonic-cli-gen/config.py.j2 | 352 ++++++++++++++++++ .../templates/sonic-cli-gen/show.py.j2 | 93 +++++ 4 files changed, 459 insertions(+), 2 deletions(-) create mode 100644 sonic-utilities-data/templates/sonic-cli-gen/common.j2 create mode 100644 sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 create mode 100644 sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 diff --git a/sonic-utilities-data/debian/install b/sonic-utilities-data/debian/install index 82d087d54d..1f67b78c20 100644 --- a/sonic-utilities-data/debian/install +++ b/sonic-utilities-data/debian/install @@ -1,2 +1,3 @@ -bash_completion.d/ /etc/ -templates/*.j2 /usr/share/sonic/templates/ +bash_completion.d/ /etc/ +templates/*.j2 /usr/share/sonic/templates/ +templates/sonic-cli-gen/*.j2 /usr/share/sonic/templates/sonic-cli-gen/ diff --git a/sonic-utilities-data/templates/sonic-cli-gen/common.j2 b/sonic-utilities-data/templates/sonic-cli-gen/common.j2 new file mode 100644 index 0000000000..51ebaa004c --- /dev/null +++ b/sonic-utilities-data/templates/sonic-cli-gen/common.j2 @@ -0,0 +1,11 @@ +{% macro make_cli_name(name) -%} +{{ name|lower|replace("_", "-") }} +{%- endmacro %} + +{% macro key_converter(keys) %} +{%- if keys|length > 1 %} + ({{ keys|map(attribute="name")|join(",") }}) +{%- else %} + {{ keys|map(attribute="name")|first }} +{%- endif %} +{% endmacro %} diff --git a/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 b/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 new file mode 100644 index 0000000000..8536d33303 --- /dev/null +++ b/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 @@ -0,0 +1,352 @@ +{%- from "common.j2" import make_cli_name, key_converter -%} +""" Autogenerated config CLI plugin """ + +import click +from config import config_mgmt +import utilities_common.cli as clicommon + + +def exit_cli(*args, **kwargs): + """ Print a message and abort CLI. """ + + click.secho(*args, **kwargs) + raise click.Abort() + +def validate_config_or_raise(cfg): + """ Validate config db data using ConfigMgmt """ + + try: + config_mgmt.ConfigMgmt().loadData(cfg) + except Exception as err: + raise Exception('Failed to validate configuration: {}'.format(err)) + + +def mod_entry_validated(db, table, key, data): + """ Modify existing entry and validate configuration """ + + cfg = db.get_config() + cfg.setdefault(table, {}) + cfg[table].setdefault(key, {}) + cfg[table][key].update(data) + + validate_config_or_raise(cfg) + db.mod_entry(table, key, data) + + +def add_entry_validated(db, table, key, data): + """ Add new entry in table and validate configuration""" + + cfg = db.get_config() + cfg.setdefault(table, {}) + if key in cfg[table]: + raise Exception(f"{key} already exists") + cfg[table][key] = data + + validate_config_or_raise(cfg) + db.set_entry(table, key, data) + + +def update_entry_validated(db, table, key, data): + """ Update entry in table and validate configuration""" + + cfg = db.get_config() + cfg.setdefault(table, {}) + if key not in cfg[table]: + raise Exception(f"{key} does not exist") + cfg[table][key].update(data) + + validate_config_or_raise(cfg) + db.mod_entry(table, key, data) + + +def del_entry_validated(db, table, key): + """ Delete entry in table and validate configuration """ + + cfg = db.get_config() + cfg.setdefault(table, {}) + if key not in cfg[table]: + raise Exception(f"{key} does not exist") + cfg[table].pop(key) + + validate_config_or_raise(cfg) + db.set_entry(table, key, None) + + +def add_list_entry_validated(db, table, key, attr, data): + """ Add new entry into list in table and validate configuration""" + + cfg = db.get_config() + cfg.setdefault(table, {}) + if key not in cfg[table]: + raise Exception(f"{key} does not exist") + cfg[table][key].setdefault(attr, []) + for entry in data: + if entry in cfg[table][key][attr]: + raise Exception(f"{entry} already exists") + cfg[table][key][attr].append(entry) + + validate_config_or_raise(cfg) + db.set_entry(table, key, {attr: cfg[table][key][attr]}) + + +def del_list_entry_validated(db, table, key, attr, data): + """ Delete entry from list in table and validate configuration""" + + cfg = db.get_config() + cfg.setdefault(table, {}) + if key not in cfg[table]: + raise Exception(f"{key} does not exist") + cfg[table][key].setdefault(attr, []) + for entry in data: + if entry not in cfg[table][key][attr]: + raise Exception(f"{entry} does not exist") + cfg[table][key][attr].remove(entry) + + validate_config_or_raise(cfg) + db.set_entry(table, key, {attr: cfg[table][key][attr]}) + + +{% macro config_object_list_update(table, object, list) %} +@{{ table.name }}.group(name="{{ make_cli_name(list.name) }}") +def {{ table.name }}_{{ list.name }}(): + """ Add/Remove {{ list.name }} in {{ table.name }} """ + + pass + +@{{ table.name }}_{{ list.name }}.command(name="add") +{%- for key in object["keys"] %} +@click.argument("{{ make_cli_name(key.name) }}") +{%- endfor %} +@click.argument("{{ make_cli_name(list.name) }}", nargs=-1) +@clicommon.pass_db +def {{ table.name }}_{{ list.name }}_add(db, + {{ object["keys"]|map(attribute="name")|join(",") }}, + {{ list.name|lower }}): + """ Add {{ list.name }} in {{ table.name }} """ + + table = "{{ table.name }}" + key = {{ key_converter(object["keys"]) }} + attr = "{{ list.name }}" + data = {{ list.name|lower }} + + try: + add_list_entry_validated(db.cfgdb, table, key, attr, data) + except Exception as err: + exit_cli(f"Error: {err}", fg="red") + +@{{ table.name }}_{{ list.name }}.command(name="remove") +{%- for key in object["keys"] %} +@click.argument("{{ make_cli_name(key.name) }}") +{%- endfor %} +@click.argument("{{ make_cli_name(list.name) }}", nargs=-1) +@clicommon.pass_db +def {{ table.name }}_{{ list.name }}_remove(db, + {{ object["keys"]|map(attribute="name")|join(",") }}, + {{ list.name|lower }}): + """ Remove {{ list.name }} in {{ table.name }} """ + + table = "{{ table.name }}" + key = {{ key_converter(object["keys"]) }} + attr = "{{ list.name }}" + data = {{ list.name|lower }} + + try: + del_list_entry_validated(db.cfgdb, table, key, attr, data) + except Exception as err: + exit_cli(f"Error: {err}", fg="red") +{% endmacro %} + + +{% macro config_object_list_update_all(table, object) %} +{% for list in object.lists %} +{{ config_object_list_update(table, object, list) }} +{% endfor %} +{% endmacro %} + + +{% macro config_static_object_attr(table, object, attr) %} +@{{ table.name }}_{{ object.name }}.command(name="{{ make_cli_name(attr.name) }}") +@click.argument("{{ make_cli_name(attr.name) }}") +@clicommon.pass_db +def {{ table.name }}_{{ object.name }}_{{ attr.name }}(db, {{ attr.name|lower }}): + """ {{ attr.description|default("") }} """ + + table = "{{ table.name }}" + key = "{{ object.name }}" + data = { + "{{ attr.name }}": {{ attr.name|lower }}, + } + try: + mod_entry_validated(db.cfgdb, table, key, data) + except Exception as err: + exit_cli(f"Error: {err}", fg="red") +{% endmacro %} + + +{# Static objects config CLI generation +E.g: + @TABLE.group(name="object") + def TABLE_object(db): +#} +{% macro config_static_object(table, object) %} +@{{ table.name }}.group(name="{{ make_cli_name(object.name) }}") +@clicommon.pass_db +def {{ table.name }}_{{ object.name }}(db): + """ {{ object.description|default("") }} """ + + pass + +{# Static objects attributes config CLI generation +E.g: + @TABLE_object.command(name="attribute") + def TABLE_object_attribute(db, attribute): +#} +{% for attr in object.attrs %} +{{ config_static_object_attr(table, object, attr) }} +{% endfor %} + +{{ config_object_list_update_all(table, object) }} +{% endmacro %} + +{# Dynamic objects config CLI generation #} + +{# Dynamic objects add command +E.g: + @TABLE.command(name="add") + @click.argument("key1") + @click.argument("key2") + @click.option("--attr1") + @click.option("--attr2") + @click.option("--attr3") + def TABLE_object_add(db, key1, key2, attr1, attr2, attr3): +#} +{% macro config_dynamic_object_add(table, object) %} +@{{ table.name }}.command(name="add") +{%- for key in object["keys"] %} +@click.argument("{{ make_cli_name(key.name) }}") +{%- endfor %} +{%- for attr in object.attrs + object.lists %} +@click.option("--{{ make_cli_name(attr.name) }}") +{%- endfor %} +@clicommon.pass_db +def {{ table.name }}_add(db, + {{ object["keys"]|map(attribute="name")|join(",") }}, + {{ (object.attrs + object.lists)|map(attribute="name")|join(",") }}): + """ Add object in {{ table.name }}. """ + + table = "{{ table.name }}" + key = {{ key_converter(object["keys"]) }} + data = {} +{%- for attr in object.attrs %} + if {{ attr.name|lower }} is not None: + data["{{ attr.name }}"] = {{ attr.name|lower }} +{%- endfor %} +{%- for list in object.lists %} + if {{ list.name|lower }} is not None: + data["{{ list.name }}"] = {{ list.name|lower }}.split(",") +{%- endfor %} + + try: + add_entry_validated(db.cfgdb, table, key, data) + except Exception as err: + exit_cli(f"Error: {err}", fg="red") +{% endmacro %} + +{# Dynamic objects update command +E.g: + @TABLE.command(name="update") + @click.argument("key1") + @click.argument("key2") + @click.option("--attr1") + @click.option("--attr2") + @click.option("--attr3") + def TABLE_object_update(db, key1, key2, attr1, attr2, attr3): +#} +{% macro config_dynamic_object_update(table, object) %} +@{{ table.name }}.command(name="update") +{%- for key in object["keys"] %} +@click.argument("{{ make_cli_name(key.name) }}") +{%- endfor %} +{%- for attr in object.attrs + object.lists %} +@click.option("--{{ make_cli_name(attr.name) }}") +{%- endfor %} +@clicommon.pass_db +def {{ table.name }}_update(db, + {{ object["keys"]|map(attribute="name")|join(",") }}, + {{ (object.attrs + object.lists)|map(attribute="name")|join(",") }}): + """ Add object in {{ table.name }}. """ + + table = "{{ table.name }}" + key = {{ key_converter(object["keys"]) }} + data = {} +{%- for attr in object.attrs %} + if {{ attr.name|lower }} is not None: + data["{{ attr.name }}"] = {{ attr.name|lower }} +{%- endfor %} +{%- for list in object.lists %} + if {{ list.name|lower }} is not None: + data["{{ list.name }}"] = {{ list.name|lower }}.split(",") +{%- endfor %} + + try: + update_entry_validated(db.cfgdb, table, key, data) + except Exception as err: + exit_cli(f"Error: {err}", fg="red") +{% endmacro %} + +{# Dynamic objects delete command +E.g: + @TABLE.command(name="delete") + @click.argument("key1") + @click.argument("key2") + def TABLE_object_add(db, key1, key2): +#} +{% macro config_dynamic_object_delete(table, object) %} +@{{ table.name }}.command(name="delete") +{%- for key in object["keys"] %} +@click.argument("{{ make_cli_name(key.name) }}") +{%- endfor %} +@clicommon.pass_db +def {{ table.name }}_add(db, + {{ object["keys"]|map(attribute="name")|join(",") }}): + """ Delete object in {{ table.name }}. """ + + table = "{{ table.name }}" + key = {{ key_converter(object["keys"]) }} + try: + del_entry_validated(db.cfg, table, key) + except Exception as err: + exit_cli(f"Error: {err}", fg="red") +{% endmacro %} + +{% macro config_dynamic_object(table, object) %} +{{ config_dynamic_object_add(table, object) }} +{{ config_dynamic_object_update(table, object) }} +{{ config_dynamic_object_delete(table, object) }} +{{ config_object_list_update_all(table, object) }} +{% endmacro %} + + +{% for table in tables %} +@click.group(name="{{ make_cli_name(table.name) }}", + cls=clicommon.AliasedGroup) +def {{ table.name }}(): + """ {{ table.description|default("") }} """ + + pass + +{% if "static-objects" in table %} +{% for object in table["static-objects"] %} +{{ config_static_object(table, object) }} +{% endfor %} +{% elif "dynamic-objects" in table %} +{% for object in table["dynamic-objects"] %} +{{ config_dynamic_object(table, object) }} +{% endfor %} +{% endif %} +{% endfor %} + +def register(cli): +{%- for table in tables %} + cli.add_command({{ table.name }}) +{%- endfor %} diff --git a/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 b/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 new file mode 100644 index 0000000000..84f76fea2f --- /dev/null +++ b/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 @@ -0,0 +1,93 @@ +import click +import tabulate +from utilities_common.db import Db +import utilities_common.cli as clicommon + +{% from "common.j2" import make_cli_name %} + +{% for table in tables %} +{% if "static-objects" in table %} +@click.group(name="{{ make_cli_name(table.name) }}", cls=clicommon.AliasedGroup) +def {{ table.name }}(): + """ {{ table.description|default('') }} """ + + pass + +{% for object in table["static-objects"] %} +@{{ table.name }}.command(name="{{ make_cli_name(object.name) }}") +@clicommon.pass_db +def {{ table.name }}_{{ object.name }}(db): + """ {{ object.description|default('') }} """ + + header = [ +{%- for attr in object.attrs %} + "{{ attr.name|upper() }}", +{%- endfor %} +{%- for list in object.lists %} + "{{ list.name|upper() }}", +{%- endfor %} + ] + body = [] + + table = db.cfgdb.get_table("{{ table.name }}") + entry = table.get("{{ object.name }}") + body.append( + [ +{%- for attr in object.attrs -%} + entry.get("{{ attr.name }}", "N/A"), +{%- endfor %} +{%- for list in object.lists -%} + "\n".join(entry.get("{{ list.name }}", [])), +{%- endfor %} + ] + ) + click.echo(tabulate.tabulate(body, header)) + +{% endfor %} +{% elif "dynamic-objects" in table %} +{% for object in table["dynamic-objects"] %} +@click.group(name="{{ make_cli_name(table.name) }}", + cls=clicommon.AliasedGroup, + invoke_without_command=True) +@clicommon.pass_db +def {{ table.name }}(db): + """ {{ object.description|default('') }} """ + + header = [ +{%- for key in object["keys"] %} + "{{ key.name|upper() }}", +{%- endfor %} +{%- for attr in object.attrs %} + "{{ attr.name|upper() }}", +{%- endfor %} +{%- for list in object.lists %} + "{{ list.name|upper() }}", +{%- endfor %} + ] + body = [] + + table = db.cfgdb.get_table("{{ table.name }}") + for key, entry in table.items(): + if not isinstance(key, tuple ): + key = (key,) + body.append( + [ + *key, +{%- for attr in object.attrs -%} + entry.get("{{ attr.name }}", "N/A"), +{%- endfor %} +{%- for list in object.lists -%} + "\n".join(entry.get("{{ list.name }}", [])), +{%- endfor %} + ] + ) + + click.echo(tabulate.tabulate(body, header)) +{% endfor %} +{% endif %} +{% endfor %} + +def register(cli): +{%- for table in tables %} + cli.add_command({{ table.name }}) +{%- endfor %} From 12bf0b27687d42c968356865301f9daa82b1c932 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Wed, 28 Apr 2021 18:17:43 +0300 Subject: [PATCH 068/173] Fix the generated function name is incorrect Signed-off-by: Stepan Blyschak --- sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 b/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 index 8536d33303..013fec2517 100644 --- a/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 +++ b/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 @@ -307,7 +307,7 @@ E.g: @click.argument("{{ make_cli_name(key.name) }}") {%- endfor %} @clicommon.pass_db -def {{ table.name }}_add(db, +def {{ table.name }}_delete(db, {{ object["keys"]|map(attribute="name")|join(",") }}): """ Delete object in {{ table.name }}. """ From 0d5dfcfc0570d69184a159045aa6c25cd0854da6 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Thu, 29 Apr 2021 00:30:47 +0300 Subject: [PATCH 069/173] Fix review comments Signed-off-by: Stepan Blyschak --- setup.py | 1 - sonic_installer/bootloader/bootloader.py | 2 +- sonic_package_manager/manager.py | 1 + sonic_package_manager/manifest.py | 1 + sonic_package_manager/registry.py | 1 + sonic_package_manager/service_creator/creator.py | 1 + sonic_package_manager/service_creator/sonic_db.py | 1 + tests/sonic_package_manager/test_constraint.py | 1 + tests/sonic_package_manager/test_manager.py | 1 + tests/sonic_package_manager/test_metadata.py | 1 + tests/sonic_package_manager/test_reference.py | 1 + tests/sonic_package_manager/test_service_creator.py | 1 + 12 files changed, 11 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index a5bbc36dfa..4cbdf52ff9 100644 --- a/setup.py +++ b/setup.py @@ -191,7 +191,6 @@ tests_require = [ 'pyfakefs', 'pytest', - 'mock>=2.0.0', 'mockredispy>=2.9.3', 'sonic-config-engine', 'deepdiff==5.2.3' diff --git a/sonic_installer/bootloader/bootloader.py b/sonic_installer/bootloader/bootloader.py index 607cd730db..a6694977ae 100644 --- a/sonic_installer/bootloader/bootloader.py +++ b/sonic_installer/bootloader/bootloader.py @@ -72,4 +72,4 @@ def get_image_path(cls, image): @contextmanager def get_path_in_image(self, image_path, path_in_image): """returns the path to the squashfs""" - yield path.join(image_path, path_in_image) \ No newline at end of file + yield path.join(image_path, path_in_image) diff --git a/sonic_package_manager/manager.py b/sonic_package_manager/manager.py index 9f3f5d95d4..ba437534ed 100644 --- a/sonic_package_manager/manager.py +++ b/sonic_package_manager/manager.py @@ -1,4 +1,5 @@ #!/usr/bin/env python + import contextlib import functools import os diff --git a/sonic_package_manager/manifest.py b/sonic_package_manager/manifest.py index 1bd5449d3a..b58a0d10f0 100644 --- a/sonic_package_manager/manifest.py +++ b/sonic_package_manager/manifest.py @@ -1,4 +1,5 @@ #!/usr/bin/env python + from abc import ABC from dataclasses import dataclass from typing import Optional, List, Dict, Any diff --git a/sonic_package_manager/registry.py b/sonic_package_manager/registry.py index bf4308efa0..8a09d9136e 100644 --- a/sonic_package_manager/registry.py +++ b/sonic_package_manager/registry.py @@ -1,4 +1,5 @@ #!/usr/bin/env python + import json from dataclasses import dataclass from typing import List, Dict diff --git a/sonic_package_manager/service_creator/creator.py b/sonic_package_manager/service_creator/creator.py index 2b305b04b3..54b9315bee 100644 --- a/sonic_package_manager/service_creator/creator.py +++ b/sonic_package_manager/service_creator/creator.py @@ -1,4 +1,5 @@ #!/usr/bin/env python + import contextlib import os import stat diff --git a/sonic_package_manager/service_creator/sonic_db.py b/sonic_package_manager/service_creator/sonic_db.py index a9ba837ab1..a064c60c4a 100644 --- a/sonic_package_manager/service_creator/sonic_db.py +++ b/sonic_package_manager/service_creator/sonic_db.py @@ -1,4 +1,5 @@ #!/usr/bin/env python + import contextlib import json import os diff --git a/tests/sonic_package_manager/test_constraint.py b/tests/sonic_package_manager/test_constraint.py index 2e7067ef63..1b34a301d2 100644 --- a/tests/sonic_package_manager/test_constraint.py +++ b/tests/sonic_package_manager/test_constraint.py @@ -1,4 +1,5 @@ #!/usr/bin/env python + from sonic_package_manager import version from sonic_package_manager.constraint import PackageConstraint from sonic_package_manager.version import Version, VersionRange diff --git a/tests/sonic_package_manager/test_manager.py b/tests/sonic_package_manager/test_manager.py index dc79f7f483..c7eb1ca7ac 100644 --- a/tests/sonic_package_manager/test_manager.py +++ b/tests/sonic_package_manager/test_manager.py @@ -1,4 +1,5 @@ #!/usr/bin/env python + from unittest.mock import Mock, call import pytest diff --git a/tests/sonic_package_manager/test_metadata.py b/tests/sonic_package_manager/test_metadata.py index 4636c18282..aee2f49428 100644 --- a/tests/sonic_package_manager/test_metadata.py +++ b/tests/sonic_package_manager/test_metadata.py @@ -1,4 +1,5 @@ #!/usr/bin/env python + import contextlib from unittest.mock import Mock, MagicMock diff --git a/tests/sonic_package_manager/test_reference.py b/tests/sonic_package_manager/test_reference.py index c986632c43..043b66ddd5 100644 --- a/tests/sonic_package_manager/test_reference.py +++ b/tests/sonic_package_manager/test_reference.py @@ -1,4 +1,5 @@ #!/usr/bin/env python + import pytest from sonic_package_manager.reference import PackageReference diff --git a/tests/sonic_package_manager/test_service_creator.py b/tests/sonic_package_manager/test_service_creator.py index c540259b49..fec8de600c 100644 --- a/tests/sonic_package_manager/test_service_creator.py +++ b/tests/sonic_package_manager/test_service_creator.py @@ -1,4 +1,5 @@ #!/usr/bin/env python + import os from unittest.mock import Mock, MagicMock From c8eaa66120d6ee13974db530064cd3e4127fc0ff Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Thu, 29 Apr 2021 18:18:04 +0300 Subject: [PATCH 070/173] update and enhance templates with multiple lists in container Signed-off-by: Stepan Blyschak --- .../templates/sonic-cli-gen/common.j2 | 10 +- .../templates/sonic-cli-gen/config.py.j2 | 234 ++++++++++-------- .../templates/sonic-cli-gen/show.py.j2 | 66 +++-- 3 files changed, 179 insertions(+), 131 deletions(-) diff --git a/sonic-utilities-data/templates/sonic-cli-gen/common.j2 b/sonic-utilities-data/templates/sonic-cli-gen/common.j2 index 51ebaa004c..3b83ee5635 100644 --- a/sonic-utilities-data/templates/sonic-cli-gen/common.j2 +++ b/sonic-utilities-data/templates/sonic-cli-gen/common.j2 @@ -1,11 +1,3 @@ -{% macro make_cli_name(name) -%} +{% macro cli_name(name) -%} {{ name|lower|replace("_", "-") }} {%- endmacro %} - -{% macro key_converter(keys) %} -{%- if keys|length > 1 %} - ({{ keys|map(attribute="name")|join(",") }}) -{%- else %} - {{ keys|map(attribute="name")|first }} -{%- endif %} -{% endmacro %} diff --git a/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 b/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 index 013fec2517..541c083d2e 100644 --- a/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 +++ b/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 @@ -1,4 +1,4 @@ -{%- from "common.j2" import make_cli_name, key_converter -%} +{%- from "common.j2" import cli_name -%} """ Autogenerated config CLI plugin """ import click @@ -12,6 +12,7 @@ def exit_cli(*args, **kwargs): click.secho(*args, **kwargs) raise click.Abort() + def validate_config_or_raise(cfg): """ Validate config db data using ConfigMgmt """ @@ -101,54 +102,91 @@ def del_list_entry_validated(db, table, key, attr, data): if entry not in cfg[table][key][attr]: raise Exception(f"{entry} does not exist") cfg[table][key][attr].remove(entry) + if not cfg[table][key][attr]: + cfg[table][key].pop(attr) validate_config_or_raise(cfg) db.set_entry(table, key, {attr: cfg[table][key][attr]}) +{%- macro gen_click_arguments(args) -%} +{%- for arg in args %} +@click.argument( + "{{ cli_name(arg.name) }}", + nargs={% if arg.is_list %}-1{% else %}1{% endif %}, +) +{%- endfor %} +{%- endmacro %} + +{%- macro gen_click_options(opts) -%} +{%- for opt in opts %} +@click.option( + "--{{ cli_name(opt.name) }}", +) +{%- endfor %} +{%- endmacro %} -{% macro config_object_list_update(table, object, list) %} -@{{ table.name }}.group(name="{{ make_cli_name(list.name) }}") -def {{ table.name }}_{{ list.name }}(): - """ Add/Remove {{ list.name }} in {{ table.name }} """ +{% macro pythonize(attrs) -%} +{{ attrs|map(attribute="name")|map("lower")|map("replace", "-", "_")|join(", ") }} +{%- endmacro %} + +{% macro config_object_list_update(group, table, object, attr) %} +{% set list_update_group = group + "_" + attr.name %} + +@{{ group }}.group(name="{{ cli_name(attr.name) }}") +def {{ list_update_group }}(): + """ Add/Delete {{ attr.name }} in {{ table.name }} """ pass -@{{ table.name }}_{{ list.name }}.command(name="add") -{%- for key in object["keys"] %} -@click.argument("{{ make_cli_name(key.name) }}") -{%- endfor %} -@click.argument("{{ make_cli_name(list.name) }}", nargs=-1) +{# Add entries to list attribute config CLI generation +E.g: + @TABLE_object.command(name="add") + @click.argument("key1", nargs=1) + @click.argument("key2", nargs=1) + @click.argument("attribute", nargs=-1) + def TABLE_object_attribute_add(db, key1, key2, attribute): +#} +@{{ list_update_group }}.command(name="add") +{{ gen_click_arguments(object["keys"] + [attr]) }} @clicommon.pass_db -def {{ table.name }}_{{ list.name }}_add(db, - {{ object["keys"]|map(attribute="name")|join(",") }}, - {{ list.name|lower }}): - """ Add {{ list.name }} in {{ table.name }} """ +def {{ list_update_group }}_add( + db, + {{ pythonize(object["keys"] + [attr]) }} +): + """ Add {{ attr.name }} in {{ table.name }} """ table = "{{ table.name }}" - key = {{ key_converter(object["keys"]) }} - attr = "{{ list.name }}" - data = {{ list.name|lower }} + key = {{ pythonize(object["keys"]) }} + attr = "{{ attr.name }}" + data = {{ pythonize([attr]) }} try: add_list_entry_validated(db.cfgdb, table, key, attr, data) except Exception as err: exit_cli(f"Error: {err}", fg="red") -@{{ table.name }}_{{ list.name }}.command(name="remove") -{%- for key in object["keys"] %} -@click.argument("{{ make_cli_name(key.name) }}") -{%- endfor %} -@click.argument("{{ make_cli_name(list.name) }}", nargs=-1) + +{# Delete entries from list attribute config CLI generation +E.g: + @TABLE_object.command(name="delete") + @click.argument("key1", nargs=1) + @click.argument("key2", nargs=1) + @click.argument("attribute", nargs=-1) + def TABLE_object_attribute_delete(db, key1, key2, attribute): +#} +@{{ list_update_group }}.command(name="delete") +{{ gen_click_arguments(object["keys"] + [attr]) }} @clicommon.pass_db -def {{ table.name }}_{{ list.name }}_remove(db, - {{ object["keys"]|map(attribute="name")|join(",") }}, - {{ list.name|lower }}): - """ Remove {{ list.name }} in {{ table.name }} """ +def {{ list_update_group }}_delete( + db, + {{ pythonize(object["keys"] + [attr]) }} +): + """ Delete {{ attr.name }} in {{ table.name }} """ table = "{{ table.name }}" - key = {{ key_converter(object["keys"]) }} - attr = "{{ list.name }}" - data = {{ list.name|lower }} + key = {{ pythonize(object["keys"]) }} + attr = "{{ attr.name }}" + data = {{ pythonize([attr]) }} try: del_list_entry_validated(db.cfgdb, table, key, attr, data) @@ -157,24 +195,26 @@ def {{ table.name }}_{{ list.name }}_remove(db, {% endmacro %} -{% macro config_object_list_update_all(table, object) %} -{% for list in object.lists %} -{{ config_object_list_update(table, object, list) }} +{% macro config_object_list_update_all(group, table, object) %} +{% for attr in object.attrs %} +{% if attr.is_list %} +{{ config_object_list_update(group, table, object, attr) }} +{% endif %} {% endfor %} {% endmacro %} {% macro config_static_object_attr(table, object, attr) %} -@{{ table.name }}_{{ object.name }}.command(name="{{ make_cli_name(attr.name) }}") -@click.argument("{{ make_cli_name(attr.name) }}") +@{{ table.name }}_{{ object.name }}.command(name="{{ cli_name(attr.name) }}") +{{ gen_click_arguments([attr]) }} @clicommon.pass_db -def {{ table.name }}_{{ object.name }}_{{ attr.name }}(db, {{ attr.name|lower }}): - """ {{ attr.description|default("") }} """ +def {{ table.name }}_{{ object.name }}_{{ attr.name }}(db, {{ pythonize([attr]) }}): + """ {{ attr.description }} """ table = "{{ table.name }}" key = "{{ object.name }}" data = { - "{{ attr.name }}": {{ attr.name|lower }}, + "{{ attr.name }}": {{ pythonize([attr]) }}, } try: mod_entry_validated(db.cfgdb, table, key, data) @@ -189,10 +229,10 @@ E.g: def TABLE_object(db): #} {% macro config_static_object(table, object) %} -@{{ table.name }}.group(name="{{ make_cli_name(object.name) }}") +@{{ table.name }}.group(name="{{ cli_name(object.name) }}") @clicommon.pass_db def {{ table.name }}_{{ object.name }}(db): - """ {{ object.description|default("") }} """ + """ {{ object.description }} """ pass @@ -205,7 +245,7 @@ E.g: {{ config_static_object_attr(table, object, attr) }} {% endfor %} -{{ config_object_list_update_all(table, object) }} +{{ config_object_list_update_all(table.name + "_" + object.name, table, object) }} {% endmacro %} {# Dynamic objects config CLI generation #} @@ -218,32 +258,26 @@ E.g: @click.option("--attr1") @click.option("--attr2") @click.option("--attr3") - def TABLE_object_add(db, key1, key2, attr1, attr2, attr3): + def TABLE_TABLE_LIST_add(db, key1, key2, attr1, attr2, attr3): #} -{% macro config_dynamic_object_add(table, object) %} -@{{ table.name }}.command(name="add") -{%- for key in object["keys"] %} -@click.argument("{{ make_cli_name(key.name) }}") -{%- endfor %} -{%- for attr in object.attrs + object.lists %} -@click.option("--{{ make_cli_name(attr.name) }}") -{%- endfor %} +{% macro config_dynamic_object_add(group, table, object) %} +@{{ group }}.command(name="add") +{{ gen_click_arguments(object["keys"]) }} +{{ gen_click_options(object.attrs) }} @clicommon.pass_db -def {{ table.name }}_add(db, - {{ object["keys"]|map(attribute="name")|join(",") }}, - {{ (object.attrs + object.lists)|map(attribute="name")|join(",") }}): +def {{ group }}_add(db, {{ pythonize(object["keys"] + object.attrs) }}): """ Add object in {{ table.name }}. """ table = "{{ table.name }}" - key = {{ key_converter(object["keys"]) }} + key = {{ pythonize(object["keys"]) }} data = {} {%- for attr in object.attrs %} - if {{ attr.name|lower }} is not None: - data["{{ attr.name }}"] = {{ attr.name|lower }} -{%- endfor %} -{%- for list in object.lists %} - if {{ list.name|lower }} is not None: - data["{{ list.name }}"] = {{ list.name|lower }}.split(",") + if {{ pythonize([attr]) }} is not None: +{%- if not attr.is_list %} + data["{{ attr.name }}"] = {{ pythonize([attr]) }} +{%- else %} + data["{{ attr.name }}"] = {{ pythonize([attr]) }}.split(",") +{%- endif %} {%- endfor %} try: @@ -260,32 +294,26 @@ E.g: @click.option("--attr1") @click.option("--attr2") @click.option("--attr3") - def TABLE_object_update(db, key1, key2, attr1, attr2, attr3): + def TABLE_TABLE_LIST_update(db, key1, key2, attr1, attr2, attr3): #} -{% macro config_dynamic_object_update(table, object) %} -@{{ table.name }}.command(name="update") -{%- for key in object["keys"] %} -@click.argument("{{ make_cli_name(key.name) }}") -{%- endfor %} -{%- for attr in object.attrs + object.lists %} -@click.option("--{{ make_cli_name(attr.name) }}") -{%- endfor %} +{% macro config_dynamic_object_update(group, table, object) %} +@{{ group }}.command(name="update") +{{ gen_click_arguments(object["keys"]) }} +{{ gen_click_options(object.attrs) }} @clicommon.pass_db -def {{ table.name }}_update(db, - {{ object["keys"]|map(attribute="name")|join(",") }}, - {{ (object.attrs + object.lists)|map(attribute="name")|join(",") }}): +def {{ group }}_update(db, {{ pythonize(object["keys"] + object.attrs) }}): """ Add object in {{ table.name }}. """ table = "{{ table.name }}" - key = {{ key_converter(object["keys"]) }} + key = {{ pythonize(object["keys"]) }} data = {} {%- for attr in object.attrs %} - if {{ attr.name|lower }} is not None: - data["{{ attr.name }}"] = {{ attr.name|lower }} -{%- endfor %} -{%- for list in object.lists %} - if {{ list.name|lower }} is not None: - data["{{ list.name }}"] = {{ list.name|lower }}.split(",") + if {{ pythonize([attr]) }} is not None: +{%- if not attr.is_list %} + data["{{ attr.name }}"] = {{ pythonize([attr]) }} +{%- else %} + data["{{ attr.name }}"] = {{ pythonize([attr]) }}.split(",") +{%- endif %} {%- endfor %} try: @@ -299,20 +327,17 @@ E.g: @TABLE.command(name="delete") @click.argument("key1") @click.argument("key2") - def TABLE_object_add(db, key1, key2): + def TABLE_TABLE_LIST_delete(db, key1, key2): #} -{% macro config_dynamic_object_delete(table, object) %} -@{{ table.name }}.command(name="delete") -{%- for key in object["keys"] %} -@click.argument("{{ make_cli_name(key.name) }}") -{%- endfor %} +{% macro config_dynamic_object_delete(group, table, object) %} +@{{ group }}.command(name="delete") +{{ gen_click_arguments(object["keys"]) }} @clicommon.pass_db -def {{ table.name }}_delete(db, - {{ object["keys"]|map(attribute="name")|join(",") }}): +def {{ group }}_delete(db, {{ pythonize(object["keys"]) }}): """ Delete object in {{ table.name }}. """ table = "{{ table.name }}" - key = {{ key_converter(object["keys"]) }} + key = {{ pythonize(object["keys"]) }} try: del_entry_validated(db.cfg, table, key) except Exception as err: @@ -320,27 +345,40 @@ def {{ table.name }}_delete(db, {% endmacro %} {% macro config_dynamic_object(table, object) %} -{{ config_dynamic_object_add(table, object) }} -{{ config_dynamic_object_update(table, object) }} -{{ config_dynamic_object_delete(table, object) }} -{{ config_object_list_update_all(table, object) }} +{# Generate another nesting group in case table holds two types of objects #} +{% if table.dynamic_objects|length > 1 %} +{% set group = table.name + "_" + object.name %} +@{{ table.name }}.group(name="{{ cli_name(object.name) }}", + cls=clicommon.AliasedGroup) +def {{ group }}(): + """ {{ object.description }} """ + + pass +{% else %} +{% set group = table.name %} +{% endif %} + +{{ config_dynamic_object_add(group, table, object) }} +{{ config_dynamic_object_update(group, table, object) }} +{{ config_dynamic_object_delete(group, table, object) }} +{{ config_object_list_update_all(group, table, object) }} {% endmacro %} {% for table in tables %} -@click.group(name="{{ make_cli_name(table.name) }}", +@click.group(name="{{ cli_name(table.name) }}", cls=clicommon.AliasedGroup) def {{ table.name }}(): - """ {{ table.description|default("") }} """ + """ {{ table.description }} """ pass -{% if "static-objects" in table %} -{% for object in table["static-objects"] %} +{% if "static_objects" in table %} +{% for object in table.static_objects %} {{ config_static_object(table, object) }} {% endfor %} -{% elif "dynamic-objects" in table %} -{% for object in table["dynamic-objects"] %} +{% elif "dynamic_objects" in table %} +{% for object in table.dynamic_objects %} {{ config_dynamic_object(table, object) }} {% endfor %} {% endif %} @@ -350,3 +388,5 @@ def register(cli): {%- for table in tables %} cli.add_command({{ table.name }}) {%- endfor %} + +{{ tables|map(attribute="name")|first }}() \ No newline at end of file diff --git a/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 b/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 index 84f76fea2f..880393f66e 100644 --- a/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 +++ b/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 @@ -1,30 +1,28 @@ +{% from "common.j2" import cli_name -%} +""" Autogenerated show CLI plugin """ + import click import tabulate -from utilities_common.db import Db import utilities_common.cli as clicommon -{% from "common.j2" import make_cli_name %} {% for table in tables %} -{% if "static-objects" in table %} -@click.group(name="{{ make_cli_name(table.name) }}", cls=clicommon.AliasedGroup) +{% if "static_objects" in table %} +@click.group(name="{{ cli_name(table.name) }}", cls=clicommon.AliasedGroup) def {{ table.name }}(): - """ {{ table.description|default('') }} """ + """ {{ table.description }}""" pass -{% for object in table["static-objects"] %} -@{{ table.name }}.command(name="{{ make_cli_name(object.name) }}") +{% for object in table.static_objects %} +@{{ table.name }}.command(name="{{ cli_name(object.name) }}") @clicommon.pass_db def {{ table.name }}_{{ object.name }}(db): - """ {{ object.description|default('') }} """ + """ {{ object.description }} """ header = [ {%- for attr in object.attrs %} "{{ attr.name|upper() }}", -{%- endfor %} -{%- for list in object.lists %} - "{{ list.name|upper() }}", {%- endfor %} ] body = [] @@ -34,24 +32,42 @@ def {{ table.name }}_{{ object.name }}(db): body.append( [ {%- for attr in object.attrs -%} +{%- if not attr.is_list %} entry.get("{{ attr.name }}", "N/A"), -{%- endfor %} -{%- for list in object.lists -%} - "\n".join(entry.get("{{ list.name }}", [])), +{%- else %} + "\n".join(entry.get("{{ attr.name }}", [])), +{%- endif %} {%- endfor %} ] ) click.echo(tabulate.tabulate(body, header)) {% endfor %} -{% elif "dynamic-objects" in table %} -{% for object in table["dynamic-objects"] %} -@click.group(name="{{ make_cli_name(table.name) }}", +{% elif "dynamic_objects" in table %} +{% if table.dynamic_objects|length > 1 %} +@click.group(name="{{ cli_name(table.name) }}", + cls=clicommon.AliasedGroup) +def {{ table.name }}(): + """ {{ table.description }} """ + + pass +{% endif %} +{% for object in table.dynamic_objects %} +{# Generate another nesting group in case table holds two types of objects #} +{% if table.dynamic_objects|length > 1 %} +{% set group = table.name %} +{% set name = object.name %} +{% else %} +{% set group = "click" %} +{% set name = table.name %} +{% endif %} + +@{{ group }}.group(name="{{ cli_name(name) }}", cls=clicommon.AliasedGroup, invoke_without_command=True) @clicommon.pass_db -def {{ table.name }}(db): - """ {{ object.description|default('') }} """ +def {{ name }}(db): + """ {{ object.description }} """ header = [ {%- for key in object["keys"] %} @@ -59,9 +75,6 @@ def {{ table.name }}(db): {%- endfor %} {%- for attr in object.attrs %} "{{ attr.name|upper() }}", -{%- endfor %} -{%- for list in object.lists %} - "{{ list.name|upper() }}", {%- endfor %} ] body = [] @@ -74,10 +87,11 @@ def {{ table.name }}(db): [ *key, {%- for attr in object.attrs -%} +{%- if not attr.is_list %} entry.get("{{ attr.name }}", "N/A"), -{%- endfor %} -{%- for list in object.lists -%} - "\n".join(entry.get("{{ list.name }}", [])), +{%- else %} + "\n".join(entry.get("{{ attr.name }}", [])), +{%- endif %} {%- endfor %} ] ) @@ -91,3 +105,5 @@ def register(cli): {%- for table in tables %} cli.add_command({{ table.name }}) {%- endfor %} + +{{ tables|map(attribute="name")|first }}() \ No newline at end of file From dd29e33a0150d0c9e1323f4d565b0e0acf1ff0a9 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Thu, 29 Apr 2021 19:37:10 +0300 Subject: [PATCH 071/173] fix cli templates Signed-off-by: Stepan Blyschak --- .../templates/sonic-cli-gen/config.py.j2 | 15 ++++++++------- .../templates/sonic-cli-gen/show.py.j2 | 2 -- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 b/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 index 541c083d2e..e7ff755a37 100644 --- a/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 +++ b/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 @@ -87,7 +87,7 @@ def add_list_entry_validated(db, table, key, attr, data): cfg[table][key][attr].append(entry) validate_config_or_raise(cfg) - db.set_entry(table, key, {attr: cfg[table][key][attr]}) + db.set_entry(table, key, cfg[table][key]) def del_list_entry_validated(db, table, key, attr, data): @@ -106,13 +106,14 @@ def del_list_entry_validated(db, table, key, attr, data): cfg[table][key].pop(attr) validate_config_or_raise(cfg) - db.set_entry(table, key, {attr: cfg[table][key][attr]}) + db.set_entry(table, key, cfg[table][key]) {%- macro gen_click_arguments(args) -%} {%- for arg in args %} @click.argument( "{{ cli_name(arg.name) }}", nargs={% if arg.is_list %}-1{% else %}1{% endif %}, + required=True, ) {%- endfor %} {%- endmacro %} @@ -132,7 +133,8 @@ def del_list_entry_validated(db, table, key, attr, data): {% macro config_object_list_update(group, table, object, attr) %} {% set list_update_group = group + "_" + attr.name %} -@{{ group }}.group(name="{{ cli_name(attr.name) }}") +@{{ group }}.group(name="{{ cli_name(attr.name) }}", + cls=clicommon.AliasedGroup) def {{ list_update_group }}(): """ Add/Delete {{ attr.name }} in {{ table.name }} """ @@ -229,7 +231,8 @@ E.g: def TABLE_object(db): #} {% macro config_static_object(table, object) %} -@{{ table.name }}.group(name="{{ cli_name(object.name) }}") +@{{ table.name }}.group(name="{{ cli_name(object.name) }}", + cls=clicommon.AliasedGroup) @clicommon.pass_db def {{ table.name }}_{{ object.name }}(db): """ {{ object.description }} """ @@ -339,7 +342,7 @@ def {{ group }}_delete(db, {{ pythonize(object["keys"]) }}): table = "{{ table.name }}" key = {{ pythonize(object["keys"]) }} try: - del_entry_validated(db.cfg, table, key) + del_entry_validated(db.cfgdb, table, key) except Exception as err: exit_cli(f"Error: {err}", fg="red") {% endmacro %} @@ -388,5 +391,3 @@ def register(cli): {%- for table in tables %} cli.add_command({{ table.name }}) {%- endfor %} - -{{ tables|map(attribute="name")|first }}() \ No newline at end of file diff --git a/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 b/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 index 880393f66e..a0c51502b3 100644 --- a/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 +++ b/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 @@ -105,5 +105,3 @@ def register(cli): {%- for table in tables %} cli.add_command({{ table.name }}) {%- endfor %} - -{{ tables|map(attribute="name")|first }}() \ No newline at end of file From e91ff811a339f8a9aef285d751620849ace7257d Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Fri, 30 Apr 2021 04:42:29 +0300 Subject: [PATCH 072/173] fix script Signed-off-by: Stepan Blyschak --- scripts/generate_shutdown_order.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/generate_shutdown_order.py b/scripts/generate_shutdown_order.py index db9f48d676..a9a1168a05 100644 --- a/scripts/generate_shutdown_order.py +++ b/scripts/generate_shutdown_order.py @@ -6,7 +6,7 @@ def main(): manager = PackageManager.get_manager() - installed_packages = manager.get_installed_packages_list() + installed_packages = manager.get_installed_packages() print('installed packages {}'.format(installed_packages)) manager.service_creator.generate_shutdown_sequence_files(installed_packages) print('Done.') From 62e26463ae4bfc333addd792b6c3738da99de755 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Fri, 30 Apr 2021 04:52:05 +0300 Subject: [PATCH 073/173] fix unresolved conftest.py Signed-off-by: Stepan Blyschak --- tests/sonic_package_manager/conftest.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/tests/sonic_package_manager/conftest.py b/tests/sonic_package_manager/conftest.py index 236376bb6b..9d5fc01b59 100644 --- a/tests/sonic_package_manager/conftest.py +++ b/tests/sonic_package_manager/conftest.py @@ -75,7 +75,6 @@ def __init__(self): components={ 'libswsscommon': Version.parse('1.0.0'), 'libsairedis': Version.parse('1.0.0') -<<<<<<< HEAD }, warm_shutdown={ 'before': ['syncd'], @@ -106,8 +105,6 @@ def __init__(self): }, fast_shutdown={ 'before': ['swss'], -======= ->>>>>>> 08337aa7637b290bb8407c38b2a5dbe3e8383b3e } ) self.add('Azure/docker-test', '1.6.0', 'test-package', '1.6.0') @@ -141,13 +138,9 @@ def from_tarball(self, filepath: str) -> Manifest: components = self.metadata_store[path][ref]['components'] return Metadata(manifest, components) -<<<<<<< HEAD def add(self, repo, reference, name, version, components=None, warm_shutdown=None, fast_shutdown=None, processes=None): -======= - def add(self, repo, reference, name, version, components=None): ->>>>>>> 08337aa7637b290bb8407c38b2a5dbe3e8383b3e repo_dict = self.metadata_store.setdefault(repo, {}) repo_dict[reference] = { 'manifest': { @@ -158,14 +151,10 @@ def add(self, repo, reference, name, version, components=None): }, 'service': { 'name': name, -<<<<<<< HEAD 'warm-shutdown': warm_shutdown or {}, 'fast-shutdown': fast_shutdown or {}, }, 'processes': processes or {} -======= - } ->>>>>>> 08337aa7637b290bb8407c38b2a5dbe3e8383b3e }, 'components': components or {}, } @@ -238,7 +227,6 @@ def fake_db(fake_metadata_resolver): add_package( content, fake_metadata_resolver, -<<<<<<< HEAD 'docker-syncd', 'latest', description='SONiC syncd service', @@ -259,8 +247,6 @@ def fake_db(fake_metadata_resolver): add_package( content, fake_metadata_resolver, -======= ->>>>>>> 08337aa7637b290bb8407c38b2a5dbe3e8383b3e 'Azure/docker-test', '1.6.0', description='SONiC Package Manager Test Package', From 79cabb201051e76cb383b7cb3a5751e619127b6b Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Fri, 30 Apr 2021 04:52:36 +0300 Subject: [PATCH 074/173] fix unresolved test_service_creator.py Signed-off-by: Stepan Blyschak --- .../test_service_creator.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/tests/sonic_package_manager/test_service_creator.py b/tests/sonic_package_manager/test_service_creator.py index 1c05b98d55..ffa6737531 100644 --- a/tests/sonic_package_manager/test_service_creator.py +++ b/tests/sonic_package_manager/test_service_creator.py @@ -28,7 +28,6 @@ def manifest(): 'dependent-of': ['swss'], 'asic-service': False, 'host-service': True, -<<<<<<< HEAD 'warm-shutdown': { 'before': ['syncd'], 'after': ['swss'], @@ -36,15 +35,12 @@ def manifest(): 'fast-shutdown': { 'before': ['swss'], }, -======= ->>>>>>> 08337aa7637b290bb8407c38b2a5dbe3e8383b3e }, 'container': { 'privileged': True, 'volumes': [ '/etc/sonic:/etc/sonic:ro' ] -<<<<<<< HEAD }, 'processes': [ { @@ -70,24 +66,12 @@ def test_service_creator(sonic_fs, manifest, package_manager, mock_feature_regis installed_packages = package_manager._get_installed_packages_and(package) creator.create(package) creator.generate_shutdown_sequence_files(installed_packages) -======= - } - }) - - -def test_service_creator(sonic_fs, manifest, mock_feature_registry, mock_sonic_db): - creator = ServiceCreator(mock_feature_registry, mock_sonic_db) - entry = PackageEntry('test', 'azure/sonic-test') - package = Package(entry, Metadata(manifest)) - creator.create(package) ->>>>>>> 08337aa7637b290bb8407c38b2a5dbe3e8383b3e assert sonic_fs.exists(os.path.join(ETC_SONIC_PATH, 'swss_dependent')) assert sonic_fs.exists(os.path.join(DOCKER_CTL_SCRIPT_LOCATION, 'test.sh')) assert sonic_fs.exists(os.path.join(SERVICE_MGMT_SCRIPT_LOCATION, 'test.sh')) assert sonic_fs.exists(os.path.join(SYSTEMD_LOCATION, 'test.service')) -<<<<<<< HEAD def read_file(name): with open(os.path.join(ETC_SONIC_PATH, name)) as file: return file.read() @@ -96,8 +80,6 @@ def read_file(name): assert read_file('fast-reboot_order') == 'teamd test swss syncd' assert read_file('test_reconcile') == 'test-process test-process-3' -======= ->>>>>>> 08337aa7637b290bb8407c38b2a5dbe3e8383b3e def test_service_creator_with_timer_unit(sonic_fs, manifest, mock_feature_registry, mock_sonic_db): creator = ServiceCreator(mock_feature_registry, mock_sonic_db) From a23fe442ba09471d2192d2f57ac02023453fe8c2 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Fri, 30 Apr 2021 05:49:13 +0300 Subject: [PATCH 075/173] fix lgtm warning Signed-off-by: Stepan Blyschak --- sonic_package_manager/service_creator/creator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sonic_package_manager/service_creator/creator.py b/sonic_package_manager/service_creator/creator.py index 125425b377..8987554ea6 100644 --- a/sonic_package_manager/service_creator/creator.py +++ b/sonic_package_manager/service_creator/creator.py @@ -5,7 +5,7 @@ import stat import subprocess from collections import defaultdict -from typing import Dict, List, Optional +from typing import Dict, Optional import jinja2 as jinja2 from prettyprinter import pformat From 16857c4e60823745872840c0c10ea8955707500f Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Fri, 30 Apr 2021 02:43:50 +0300 Subject: [PATCH 076/173] add clear list command Signed-off-by: Stepan Blyschak --- .../templates/sonic-cli-gen/config.py.j2 | 41 +++++++++++ .../templates/sonic-cli-gen/show.py.j2 | 70 ++++++++----------- 2 files changed, 72 insertions(+), 39 deletions(-) diff --git a/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 b/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 index e7ff755a37..4d70f9d1e8 100644 --- a/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 +++ b/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 @@ -41,6 +41,7 @@ def add_entry_validated(db, table, key, data): cfg.setdefault(table, {}) if key in cfg[table]: raise Exception(f"{key} already exists") + cfg[table][key] = data validate_config_or_raise(cfg) @@ -54,6 +55,7 @@ def update_entry_validated(db, table, key, data): cfg.setdefault(table, {}) if key not in cfg[table]: raise Exception(f"{key} does not exist") + cfg[table][key].update(data) validate_config_or_raise(cfg) @@ -67,6 +69,7 @@ def del_entry_validated(db, table, key): cfg.setdefault(table, {}) if key not in cfg[table]: raise Exception(f"{key} does not exist") + cfg[table].pop(key) validate_config_or_raise(cfg) @@ -108,6 +111,13 @@ def del_list_entry_validated(db, table, key, attr, data): validate_config_or_raise(cfg) db.set_entry(table, key, cfg[table][key]) + +def clear_list_entry_validated(db, table, key, attr): + """ Clear list in object and validate configuration""" + + update_entry_validated(db, table, key, {attr: []}) + + {%- macro gen_click_arguments(args) -%} {%- for arg in args %} @click.argument( @@ -194,6 +204,34 @@ def {{ list_update_group }}_delete( del_list_entry_validated(db.cfgdb, table, key, attr, data) except Exception as err: exit_cli(f"Error: {err}", fg="red") + + +{# Clear entries from list attribute config CLI generation +E.g: + @TABLE_object.command(name="delete") + @click.argument("key1", nargs=1) + @click.argument("key2", nargs=1) + def TABLE_object_attribute_clear(db, key1, key2): +#} +@{{ list_update_group }}.command(name="clear") +{{ gen_click_arguments(object["keys"]) }} +@clicommon.pass_db +def {{ list_update_group }}_delete( + db, + {{ pythonize(object["keys"]) }} +): + """ Clear {{ attr.name }} in {{ table.name }} """ + + table = "{{ table.name }}" + key = {{ pythonize(object["keys"]) }} + attr = "{{ attr.name }}" + data = {{ pythonize([attr]) }} + + try: + clear_list_entry_validated(db.cfgdb, table, key, attr) + except Exception as err: + exit_cli(f"Error: {err}", fg="red") + {% endmacro %} @@ -389,5 +427,8 @@ def {{ table.name }}(): def register(cli): {%- for table in tables %} + cli_node = {{ table.name }} + if cli_node.name in cli.commands: + raise Exception(f"{cli_node.name} already exists in CLI") cli.add_command({{ table.name }}) {%- endfor %} diff --git a/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 b/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 index a0c51502b3..191a466852 100644 --- a/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 +++ b/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 @@ -3,12 +3,29 @@ import click import tabulate +import natsort import utilities_common.cli as clicommon +{% macro print_attr(attr) %} +{%- if not attr.is_list %} +entry.get("{{ attr.name }}", "N/A") +{%- else %} +"\n".join(entry.get("{{ attr.name }}", [])) +{%- endif %} +{% endmacro %} + + +{% macro gen_header(attrs) %} +{% for attr in attrs %} +"{{ attr.name|upper|replace("_", " ")|replace("-", " ") }}", +{% endfor %} +{% endmacro %} + {% for table in tables %} {% if "static_objects" in table %} -@click.group(name="{{ cli_name(table.name) }}", cls=clicommon.AliasedGroup) +@click.group(name="{{ cli_name(table.name) }}", + cls=clicommon.AliasedGroup) def {{ table.name }}(): """ {{ table.description }}""" @@ -20,26 +37,13 @@ def {{ table.name }}(): def {{ table.name }}_{{ object.name }}(db): """ {{ object.description }} """ - header = [ -{%- for attr in object.attrs %} - "{{ attr.name|upper() }}", -{%- endfor %} - ] + header = [{{ gen_header(object.attrs) }}] body = [] table = db.cfgdb.get_table("{{ table.name }}") entry = table.get("{{ object.name }}") - body.append( - [ -{%- for attr in object.attrs -%} -{%- if not attr.is_list %} - entry.get("{{ attr.name }}", "N/A"), -{%- else %} - "\n".join(entry.get("{{ attr.name }}", [])), -{%- endif %} -{%- endfor %} - ] - ) + row = [{%- for attr in object.attrs -%} {{ print_attr(attr) }}, {%- endfor %}] + body.append(row) click.echo(tabulate.tabulate(body, header)) {% endfor %} @@ -69,32 +73,17 @@ def {{ table.name }}(): def {{ name }}(db): """ {{ object.description }} """ - header = [ -{%- for key in object["keys"] %} - "{{ key.name|upper() }}", -{%- endfor %} -{%- for attr in object.attrs %} - "{{ attr.name|upper() }}", -{%- endfor %} - ] + header = [{{ gen_header(object["keys"] + object.attrs) }}] body = [] table = db.cfgdb.get_table("{{ table.name }}") - for key, entry in table.items(): - if not isinstance(key, tuple ): + for key in natsort.natsorted(table): + entry = table[key] + if not isinstance(key, tuple): key = (key,) - body.append( - [ - *key, -{%- for attr in object.attrs -%} -{%- if not attr.is_list %} - entry.get("{{ attr.name }}", "N/A"), -{%- else %} - "\n".join(entry.get("{{ attr.name }}", [])), -{%- endif %} -{%- endfor %} - ] - ) + + row = [*key, {%- for attr in object.attrs -%} {{ print_attr(attr) }}, {%- endfor %}] + body.append(row) click.echo(tabulate.tabulate(body, header)) {% endfor %} @@ -103,5 +92,8 @@ def {{ name }}(db): def register(cli): {%- for table in tables %} + cli_node = {{ table.name }} + if cli_node.name in cli.commands: + raise Exception(f"{cli_node.name} already exists in CLI") cli.add_command({{ table.name }}) {%- endfor %} From f4be57955da555e8650b5e2e56ed10a31bc99476 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Fri, 30 Apr 2021 06:30:53 +0300 Subject: [PATCH 077/173] fix lgtm warning Signed-off-by: Stepan Blyschak --- sonic_package_manager/service_creator/creator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sonic_package_manager/service_creator/creator.py b/sonic_package_manager/service_creator/creator.py index 8987554ea6..9b4081a82d 100644 --- a/sonic_package_manager/service_creator/creator.py +++ b/sonic_package_manager/service_creator/creator.py @@ -5,7 +5,7 @@ import stat import subprocess from collections import defaultdict -from typing import Dict, Optional +from typing import Dict import jinja2 as jinja2 from prettyprinter import pformat From dfa3cccdea80965e08628d52305da946b4138c42 Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Fri, 30 Apr 2021 11:01:08 +0000 Subject: [PATCH 078/173] Parsing for static YANG models - sonic-flex_counter.yang, sonic-device_metadata.yang DONE Signed-off-by: Vadym Hlushko --- sonic_cli_gen/main.py | 2 +- sonic_cli_gen/yang_parser.py | 93 +++++++++++++++++++++++++++++------- 2 files changed, 78 insertions(+), 17 deletions(-) diff --git a/sonic_cli_gen/main.py b/sonic_cli_gen/main.py index a8b01a7034..d6826a6e7c 100644 --- a/sonic_cli_gen/main.py +++ b/sonic_cli_gen/main.py @@ -19,7 +19,7 @@ def cli(ctx): @click.pass_context def generate_config(ctx): """ List available packages """ - gen = CliGenerator('sonic-vlan') + gen = CliGenerator('sonic-device_metadata') gen.generate_config_plugin() pass diff --git a/sonic_cli_gen/yang_parser.py b/sonic_cli_gen/yang_parser.py index 77f9ded716..ba9026adec 100644 --- a/sonic_cli_gen/yang_parser.py +++ b/sonic_cli_gen/yang_parser.py @@ -1,6 +1,7 @@ #!/usr/bin/env python try: + import pdb import os import sys import pprint @@ -18,34 +19,94 @@ def __init__(self, # index of yang model inside conf_mgmt.sy.yJson object self.idx_yJson = None # 'module' entity from .yang file - self.y_module = OrderedDict() + self.y_module = None # top level 'container' entity from .yang file - self.y_top_level_container = OrderedDict() - # 'container' entities from .yang file - self.y_tables = list() + self.y_top_level_container = None + # 'container' entities from .yang file that represent Config DB table + self.y_table_containers = None # dictionary that represent Config DB schema - self.yang_2_dict = OrderedDict() + self.yang_2_dict = dict() try: self.conf_mgmt = ConfigMgmt() except Exception as e: raise Exception("Failed to load the {} class".format(str(e))) - def fail(self, e): - print(e) - raise e - def parse_yang_model(self): self._init_yang_module_and_containers() - self._determine_tables_type() + # determine how many (1 or couple) containers yang model have after 'top level container' + if isinstance(self.y_table_containers, list): + print('LIST') + for tbl_cont in self.y_table_containers: + self._fill_yang_2_dict(tbl_cont) + else: + print('NOT LIST') + self._fill_yang_2_dict(self.y_table_containers) + - def _determine_tables_type(self): - for table in self.y_tables: - if table.get('list') is None: - self.yang_2_dict[table.get('@name')] = {'type': 'static'} + def _fill_yang_2_dict(self, tbl_cont): + self.yang_2_dict['tables'] = list() + # element for self.yang_2_dict list + y2d_elem = dict() + + y2d_elem['name'] = tbl_cont.get('@name') + y2d_elem['description'] = '' + if tbl_cont.get('description') is not None: + y2d_elem['description'] = tbl_cont.get('description').get('text') + y2d_elem['dynamic-objects'] = list() + y2d_elem['static-objects'] = list() + + # determine if 'container' is a 'list' or 'static' + # 'static' means that yang model 'container' entity does NOT have a 'list' entity + if tbl_cont.get('list') is None: + # TODO write comment about objects containers inside table containers + obj_cont = tbl_cont.get('container') + if isinstance(obj_cont, list): + # flex counter + print ("FLEX") + for cont in obj_cont: + self._on_static_container(cont, y2d_elem) else: - self.yang_2_dict[table.get('@name')] = {'type': 'list'} + print ("METADATA") + # device metadata + self._on_static_container(obj_cont, y2d_elem) + else: + self._on_list_container(tbl_cont, y2d_elem) + + self.yang_2_dict['tables'].append(y2d_elem) + pdb.set_trace() + + def _on_static_container(self, cont, y2d_elem): + # element for y2d_elem['static-objects'] + static_obj_elem = dict() + static_obj_elem['name'] = cont.get('@name') + static_obj_elem['description'] = '' + if cont.get('description') is not None: + static_obj_elem['description'] = cont.get('description').get('text') + + self._parse_yang_leafs(cont.get('leaf'), static_obj_elem, y2d_elem) + + def _parse_yang_leafs(self, y_leafs, static_obj_elem, y2d_elem): + static_obj_elem['attrs'] = list() + # The YANG 'container entity may have only 1 'leaf' element or list of 'leaf' elements + if isinstance(y_leafs, list): + for leaf in y_leafs: + attr = dict() + attr['name'] = leaf.get('@name') + attr['is-leaf-list'] = leaf.get('__isleafList') + static_obj_elem['attrs'].append(attr) + + y2d_elem['static-objects'].append(static_obj_elem) + else: + attr = dict() + attr['name'] = y_leafs.get('@name') + attr['is-leaf-list'] = y_leafs.get('__isleafList') + static_obj_elem['attrs'].append(attr) + y2d_elem['static-objects'].append(static_obj_elem) + + def _on_list_container(self, cont, y2d_elem): + pass def _init_yang_module_and_containers(self): self._find_index_of_yang_model() @@ -53,7 +114,7 @@ def _init_yang_module_and_containers(self): self.y_module = self.conf_mgmt.sy.yJson[self.idx_yJson]['module'] if self.y_module.get('container') is not None: self.y_top_level_container = self.y_module['container'] - self.y_tables = self.y_top_level_container['container'] + self.y_table_containers = self.y_top_level_container['container'] else: raise KeyError('YANG model {} does NOT have "container" element'.format(self.yang_model_name)) From 6b75f4ba1bc9ba66a4ee359b61108a8ec1e45539 Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Wed, 5 May 2021 14:35:40 +0000 Subject: [PATCH 079/173] Done refactoring for parsing 'static' YANG models Signed-off-by: Vadym Hlushko --- sonic_cli_gen/__init__.py | 5 + sonic_cli_gen/generator.py | 4 +- sonic_cli_gen/main.py | 8 +- sonic_cli_gen/yang_parser.py | 243 ++++++++++++++++++++++------------- 4 files changed, 165 insertions(+), 95 deletions(-) diff --git a/sonic_cli_gen/__init__.py b/sonic_cli_gen/__init__.py index e69de29bb2..7e49cacd56 100644 --- a/sonic_cli_gen/__init__.py +++ b/sonic_cli_gen/__init__.py @@ -0,0 +1,5 @@ +#!/usr/bin/env python + +from sonic_cli_gen.generator import CliGenerator + +__all__ = ['CliGenerator'] \ No newline at end of file diff --git a/sonic_cli_gen/generator.py b/sonic_cli_gen/generator.py index 154d82409b..96d87f776c 100644 --- a/sonic_cli_gen/generator.py +++ b/sonic_cli_gen/generator.py @@ -8,7 +8,9 @@ class CliGenerator: """ SONiC CLI generator. This class provides public API for sonic-cli-gen python library. It can generate config, - show, sonic-clear commands """ + show, sonic-clear commands + """ + def __init__(self, yang_model): """ Initialize PackageManager. """ diff --git a/sonic_cli_gen/main.py b/sonic_cli_gen/main.py index d6826a6e7c..f5ba5a49b5 100644 --- a/sonic_cli_gen/main.py +++ b/sonic_cli_gen/main.py @@ -12,14 +12,14 @@ @click.pass_context def cli(ctx): """ SONiC CLI generator """ - print ("cli") pass @cli.command() +@click.argument('yang_model_name') @click.pass_context -def generate_config(ctx): - """ List available packages """ - gen = CliGenerator('sonic-device_metadata') +def generate_config(ctx, yang_model_name): + """ Generate config plugin """ + gen = CliGenerator(yang_model_name) gen.generate_config_plugin() pass diff --git a/sonic_cli_gen/yang_parser.py b/sonic_cli_gen/yang_parser.py index ba9026adec..2c5b481b98 100644 --- a/sonic_cli_gen/yang_parser.py +++ b/sonic_cli_gen/yang_parser.py @@ -11,120 +11,183 @@ raise ImportError("%s - required module not found" % str(e)) class YangParser: - """ YANG model parser """ + """ YANG model parser + + Attributes: + yang_model_name: Name of the YANG model file + conf_mgmt: Instance of Config Mgmt class to help parse YANG models + idx_yJson: Index of YANG model file (1 attr) inside conf_mgmt.sy.yJson object + y_module: Reference to 'module' entity from YANG model file + y_top_level_container: Reference to top level 'container' entity from YANG model file + y_table_containers: Reference to 'container' entities from YANG model file + that represent Config DB tables + yang_2_dict: dictionary created from YANG model file that represent Config DB schema + """ def __init__(self, yang_model_name): self.yang_model_name = yang_model_name self.conf_mgmt = None - # index of yang model inside conf_mgmt.sy.yJson object self.idx_yJson = None - # 'module' entity from .yang file self.y_module = None - # top level 'container' entity from .yang file self.y_top_level_container = None - # 'container' entities from .yang file that represent Config DB table self.y_table_containers = None - # dictionary that represent Config DB schema self.yang_2_dict = dict() try: self.conf_mgmt = ConfigMgmt() except Exception as e: raise Exception("Failed to load the {} class".format(str(e))) + + def _init_yang_module_and_containers(self): + """ Initialize inner class variables: + self.y_module + self.y_top_level_container + self.y_table_containers - def parse_yang_model(self): - self._init_yang_module_and_containers() - - # determine how many (1 or couple) containers yang model have after 'top level container' - if isinstance(self.y_table_containers, list): - print('LIST') - for tbl_cont in self.y_table_containers: - self._fill_yang_2_dict(tbl_cont) - else: - print('NOT LIST') - self._fill_yang_2_dict(self.y_table_containers) + Raises: + KeyError: if invalid YANG model provided + KeyError: if YANG models is NOT exist + """ + self._find_index_of_yang_model() - def _fill_yang_2_dict(self, tbl_cont): - self.yang_2_dict['tables'] = list() - # element for self.yang_2_dict list - y2d_elem = dict() - - y2d_elem['name'] = tbl_cont.get('@name') - y2d_elem['description'] = '' - if tbl_cont.get('description') is not None: - y2d_elem['description'] = tbl_cont.get('description').get('text') - y2d_elem['dynamic-objects'] = list() - y2d_elem['static-objects'] = list() - - # determine if 'container' is a 'list' or 'static' - # 'static' means that yang model 'container' entity does NOT have a 'list' entity - if tbl_cont.get('list') is None: - # TODO write comment about objects containers inside table containers - obj_cont = tbl_cont.get('container') - if isinstance(obj_cont, list): - # flex counter - print ("FLEX") - for cont in obj_cont: - self._on_static_container(cont, y2d_elem) + if self.idx_yJson is not None: + self.y_module = self.conf_mgmt.sy.yJson[self.idx_yJson]['module'] + if self.y_module.get('container') is not None: + self.y_top_level_container = self.y_module['container'] + self.y_table_containers = self.y_top_level_container['container'] else: - print ("METADATA") - # device metadata - self._on_static_container(obj_cont, y2d_elem) + raise KeyError('YANG model {} does NOT have "container" element'.format(self.yang_model_name)) else: - self._on_list_container(tbl_cont, y2d_elem) - - self.yang_2_dict['tables'].append(y2d_elem) - pdb.set_trace() + raise KeyError('YANG model {} is NOT exist'.format(self.yang_model_name)) - def _on_static_container(self, cont, y2d_elem): - # element for y2d_elem['static-objects'] - static_obj_elem = dict() - static_obj_elem['name'] = cont.get('@name') - static_obj_elem['description'] = '' - if cont.get('description') is not None: - static_obj_elem['description'] = cont.get('description').get('text') - - self._parse_yang_leafs(cont.get('leaf'), static_obj_elem, y2d_elem) - - def _parse_yang_leafs(self, y_leafs, static_obj_elem, y2d_elem): - static_obj_elem['attrs'] = list() - # The YANG 'container entity may have only 1 'leaf' element or list of 'leaf' elements - if isinstance(y_leafs, list): - for leaf in y_leafs: - attr = dict() - attr['name'] = leaf.get('@name') - attr['is-leaf-list'] = leaf.get('__isleafList') - static_obj_elem['attrs'].append(attr) + def _find_index_of_yang_model(self): + """ Find index of provided YANG model inside yJson object + and save it to self.idx_yJson variable + """ - y2d_elem['static-objects'].append(static_obj_elem) - else: - attr = dict() - attr['name'] = y_leafs.get('@name') - attr['is-leaf-list'] = y_leafs.get('__isleafList') - static_obj_elem['attrs'].append(attr) - y2d_elem['static-objects'].append(static_obj_elem) + for i in range(len(self.conf_mgmt.sy.yJson)): + if (self.conf_mgmt.sy.yJson[i]['module']['@name'] == self.yang_model_name): + self.idx_yJson = i - def _on_list_container(self, cont, y2d_elem): - pass + def parse_yang_model(self): + """ Parse proviced YANG model + and save output to self.yang_2_dict obj + """ - def _init_yang_module_and_containers(self): - self._find_index_of_yang_model() + self._init_yang_module_and_containers() + self.yang_2_dict['tables'] = list() - self.y_module = self.conf_mgmt.sy.yJson[self.idx_yJson]['module'] - if self.y_module.get('container') is not None: - self.y_top_level_container = self.y_module['container'] - self.y_table_containers = self.y_top_level_container['container'] + # determine how many (1 or couple) containers yang model have after 'top level container' + if isinstance(self.y_table_containers, list): + for tbl_cont in self.y_table_containers: + y2d_elem = on_table_container(tbl_cont) + self.yang_2_dict['tables'].append(y2d_elem) else: - raise KeyError('YANG model {} does NOT have "container" element'.format(self.yang_model_name)) - - # find index of yang_model inside yJson object - def _find_index_of_yang_model(self): - for i in range(len(self.conf_mgmt.sy.yJson)): - if (self.conf_mgmt.sy.yJson[i]['module']['@name'] == self.yang_model_name): - self.idx_yJson = i + y2d_elem = on_table_container(self.y_table_containers) + self.yang_2_dict['tables'].append(y2d_elem) + pdb.set_trace() - - - \ No newline at end of file +def on_table_container(tbl_cont: OrderedDict) -> dict: + """ Parse 'table' container, + 'table' container goes after 'top level' container + + Args: + tbl_cont: reference to 'table' container + Returns: + dictionary - element for self.yang_2_dict['tables'] + """ + + if tbl_cont.get('description') is not None: + description = tbl_cont.get('description').get('text') + else: + description = '' + + y2d_elem = { + 'name': tbl_cont.get('@name'), + 'description': description, + 'dynamic-objects': list(), + 'static-objects': list() + } + + # determine if 'container' is a 'list' or 'static' + # 'static' means that yang model 'container' entity does NOT have a 'list' entity + if tbl_cont.get('list') is None: + # 'object' container goes after 'table' container + # 'object' container have 2 types - list (like sonic-flex_counter.yang) and NOT list (like sonic-device_metadata.yang) + obj_cont = tbl_cont.get('container') + if isinstance(obj_cont, list): + for cont in obj_cont: + static_obj_elem = on_static_container(cont) + y2d_elem['static-objects'].append(static_obj_elem) + else: + static_obj_elem = on_static_container(obj_cont) + y2d_elem['static-objects'].append(static_obj_elem) + else: + on_list_container(tbl_cont) + + return y2d_elem + +def on_static_container(cont: OrderedDict) -> dict: + """ Parse container that does NOT have a 'list' entity ('static') + + Args: + cont: reference to 'static' container + Returns: + dictionary - element for y2d_elem['static-objects'] + """ + + if cont.get('description') is not None: + description = cont.get('description').get('text') + else: + description = '' + + static_obj_elem = { + 'name': cont.get('@name'), + 'description': description, + 'attrs': list() + } + static_obj_elem['attrs'] = parse_yang_leafs(cont.get('leaf')) + + return static_obj_elem + +def parse_yang_leafs(y_leafs) -> list: + """ Parse all the 'leafs' + + Args: + y_leafs: reference to all 'leaf' elements + Returns: + list - list of parsed 'leafs' + """ + ret_attrs_list = list() + # The YANG 'container' entity may have only 1 'leaf' element OR a list of 'leaf' elements + if isinstance(y_leafs, list): + for leaf in y_leafs: + attr = on_leaf(leaf) + ret_attrs_list.append(attr) + else: + attr = on_leaf(y_leafs) + ret_attrs_list.append(attr) + + return ret_attrs_list + +def on_leaf(leaf: OrderedDict) -> dict: + """ Parse a single 'leaf' element + + Args: + leaf: reference to a 'leaf' entity + Returns: + dictionary - parsed 'leaf' element + """ + mandatory = False + if leaf.get('mandatory') is not None: + mandatory = leaf.get('mandatory').get('@value') + + attr = { 'name': leaf.get('@name'), + 'is-leaf-list': leaf.get('__isleafList'), + 'mandatory': mandatory } + return attr + +def on_list_container(cont): + pass \ No newline at end of file From d2be52b6b24180eb870b57b60ea02d8a9a9a324f Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Thu, 6 May 2021 14:17:24 +0000 Subject: [PATCH 080/173] Added parsing for 'dynamic' YANG models, done refactoring of whole code Signed-off-by: Vadym Hlushko --- sonic_cli_gen/generator.py | 14 ++- sonic_cli_gen/main.py | 11 ++- sonic_cli_gen/yang_parser.py | 179 +++++++++++++++++++++++++---------- 3 files changed, 150 insertions(+), 54 deletions(-) diff --git a/sonic_cli_gen/generator.py b/sonic_cli_gen/generator.py index 96d87f776c..c328fb9995 100644 --- a/sonic_cli_gen/generator.py +++ b/sonic_cli_gen/generator.py @@ -8,7 +8,7 @@ class CliGenerator: """ SONiC CLI generator. This class provides public API for sonic-cli-gen python library. It can generate config, - show, sonic-clear commands + show, sonic-clear CLI plugins """ def __init__(self, @@ -18,16 +18,20 @@ def __init__(self, self.yang_model_name = yang_model def generate_config_plugin(self): + """ Generate CLI plugin for 'config' CLI group. """ parser = YangParser(self.yang_model_name) - parser.parse_yang_model() + yang_dict = parser.parse_yang_model() pass - #TODO def generate_show_plugin(self): - print ("show") + """ Generate CLI plugin for 'show' CLI group. """ + parser = YangParser(self.yang_model_name) + yang_dict = parser.parse_yang_model() pass # to be implemented in the next Phases def generate_sonic_clear_plugin(self): - print ("sonic-clear") + """ Generate CLI plugin for 'sonic-clear' CLI group. """ + parser = YangParser(self.yang_model_name) + yang_dict = parser.parse_yang_model() pass diff --git a/sonic_cli_gen/main.py b/sonic_cli_gen/main.py index f5ba5a49b5..bf783e8541 100644 --- a/sonic_cli_gen/main.py +++ b/sonic_cli_gen/main.py @@ -18,10 +18,19 @@ def cli(ctx): @click.argument('yang_model_name') @click.pass_context def generate_config(ctx, yang_model_name): - """ Generate config plugin """ + """ Generate CLI plugin (click) for 'config' CLI group. """ gen = CliGenerator(yang_model_name) gen.generate_config_plugin() pass +@cli.command() +@click.argument('yang_model_name') +@click.pass_context +def generate_show(ctx, yang_model_name): + """ Generate CLI plugin (click) for 'show' CLI group. """ + gen = CliGenerator(yang_model_name) + gen.generate_show_plugin() + pass + if __name__ == '__main__': cli() \ No newline at end of file diff --git a/sonic_cli_gen/yang_parser.py b/sonic_cli_gen/yang_parser.py index 2c5b481b98..5e1229e103 100644 --- a/sonic_cli_gen/yang_parser.py +++ b/sonic_cli_gen/yang_parser.py @@ -1,10 +1,6 @@ #!/usr/bin/env python try: - import pdb - import os - import sys - import pprint from collections import OrderedDict from config.config_mgmt import ConfigMgmt except ImportError as e: @@ -70,15 +66,19 @@ def _find_index_of_yang_model(self): if (self.conf_mgmt.sy.yJson[i]['module']['@name'] == self.yang_model_name): self.idx_yJson = i - def parse_yang_model(self): + def parse_yang_model(self) -> dict: """ Parse proviced YANG model and save output to self.yang_2_dict obj + + Returns: + dictionary - parsed YANG model in dictionary format """ self._init_yang_module_and_containers() self.yang_2_dict['tables'] = list() - # determine how many (1 or couple) containers yang model have after 'top level container' + # determine how many (1 or couple) containers YANG model have after 'top level' container + # 'table' container it is a container that goes after 'top level' container if isinstance(self.y_table_containers, list): for tbl_cont in self.y_table_containers: y2d_elem = on_table_container(tbl_cont) @@ -86,8 +86,22 @@ def parse_yang_model(self): else: y2d_elem = on_table_container(self.y_table_containers) self.yang_2_dict['tables'].append(y2d_elem) + + return self.yang_2_dict + +def get_description(y_entity: OrderedDict) -> str: + """ Parse 'description' entity from any YANG element - pdb.set_trace() + Args: + y_entity: reference to YANG 'container' OR 'list' OR 'leaf' ... + Returns: + str - text of the 'description' + """ + + if y_entity.get('description') is not None: + return y_entity.get('description').get('text') + else: + return '' def on_table_container(tbl_cont: OrderedDict) -> dict: """ Parse 'table' container, @@ -99,80 +113,150 @@ def on_table_container(tbl_cont: OrderedDict) -> dict: dictionary - element for self.yang_2_dict['tables'] """ - if tbl_cont.get('description') is not None: - description = tbl_cont.get('description').get('text') - else: - description = '' - y2d_elem = { 'name': tbl_cont.get('@name'), - 'description': description, + 'description': get_description(tbl_cont), 'dynamic-objects': list(), 'static-objects': list() } - # determine if 'container' is a 'list' or 'static' - # 'static' means that yang model 'container' entity does NOT have a 'list' entity - if tbl_cont.get('list') is None: + # determine if 'container' have a 'list' entity + tbl_cont_lists = tbl_cont.get('list') + + if tbl_cont_lists is None: + is_list = False # 'object' container goes after 'table' container # 'object' container have 2 types - list (like sonic-flex_counter.yang) and NOT list (like sonic-device_metadata.yang) obj_cont = tbl_cont.get('container') if isinstance(obj_cont, list): for cont in obj_cont: - static_obj_elem = on_static_container(cont) + static_obj_elem = on_container(cont, is_list) y2d_elem['static-objects'].append(static_obj_elem) else: - static_obj_elem = on_static_container(obj_cont) + static_obj_elem = on_container(obj_cont, is_list) y2d_elem['static-objects'].append(static_obj_elem) else: - on_list_container(tbl_cont) + is_list = True + # 'container' can have more than 1 'list' + if isinstance(tbl_cont_lists, list): + for _list in tbl_cont_lists: + dynamic_obj_elem = on_container(_list, is_list) + y2d_elem['dynamic-objects'].append(dynamic_obj_elem) + else: + dynamic_obj_elem = on_container(tbl_cont_lists, is_list) + y2d_elem['dynamic-objects'].append(dynamic_obj_elem) return y2d_elem -def on_static_container(cont: OrderedDict) -> dict: - """ Parse container that does NOT have a 'list' entity ('static') +def on_container(cont: OrderedDict, is_list: bool) -> dict: + """ Parse a 'container' that have only 'leafs' or 'list' with 'leafs' Args: - cont: reference to 'static' container + cont: reference to 'container' Returns: - dictionary - element for y2d_elem['static-objects'] + dictionary - element for y2d_elem['static-objects'] OR y2d_elem['dynamic-objects'] """ - if cont.get('description') is not None: - description = cont.get('description').get('text') - else: - description = '' - - static_obj_elem = { + obj_elem = { 'name': cont.get('@name'), - 'description': description, + 'description': get_description(cont), 'attrs': list() } - static_obj_elem['attrs'] = parse_yang_leafs(cont.get('leaf')) - return static_obj_elem + if is_list: + obj_elem['key'] = cont.get('key').get('@value') + + attrs_list = list() + + if cont.get('leaf') is not None: + is_leaf_list = False + ret_leafs = on_leafs(cont.get('leaf'), is_leaf_list) + attrs_list.extend(ret_leafs) + + if cont.get('leaf-list') is not None: + is_leaf_list = True + ret_leaf_lists = on_leafs(cont.get('leaf-list'), is_leaf_list) + attrs_list.extend(ret_leaf_lists) + + if cont.get('choice') is not None: + y_choices = cont.get('choice') + ret_choice_leafs = on_choices(y_choices) + attrs_list.extend(ret_choice_leafs) + + obj_elem['attrs'] = attrs_list + + return obj_elem + +def on_choices(y_choices) -> list: + """ Parse a YANG 'choice' entities -def parse_yang_leafs(y_leafs) -> list: - """ Parse all the 'leafs' + Args: + cont: reference to 'choice' + Returns: + dictionary - element for obj_elem['attrs'], 'attrs' contain a parsed 'leafs' + """ + + ret_attrs = list() + + # the YANG model can have multiple 'choice' entities inside 'container' or 'list' + if isinstance(y_choices, list): + for choice in y_choices: + attrs = on_choice_cases(choice.get('case')) + ret_attrs.extend(attrs) + else: + ret_attrs = on_choice_cases(y_choices.get('case')) + + return ret_attrs + +def on_choice_cases(y_cases: list) -> list: + """ Parse a single YANG 'case' entity from 'choice' entity + + Args: + cont: reference to 'case' + Returns: + dictionary - element for obj_elem['attrs'], 'attrs' contain a parsed 'leafs' + """ + + ret_attrs = list() + + if isinstance(y_cases, list): + for case in y_cases: + if case.get('leaf') is not None: + is_leaf_list = False + ret_leafs = on_leafs(case.get('leaf'), is_leaf_list) + ret_attrs.extend(ret_leafs) + + if case.get('leaf-list') is not None: + is_leaf_list = True + ret_leaf_lists = on_leafs(case.get('leaf-list'), is_leaf_list) + ret_attrs.extend(ret_leaf_lists) + else: + raise Exception('It has no sense to using a single "case" element inside "choice" element') + + return ret_attrs + +def on_leafs(y_leafs, is_leaf_list: bool) -> list: + """ Parse all the 'leaf' or 'leaf-list' elements Args: y_leafs: reference to all 'leaf' elements Returns: - list - list of parsed 'leafs' + list - list of parsed 'leaf' elements """ - ret_attrs_list = list() + + ret_attrs = list() # The YANG 'container' entity may have only 1 'leaf' element OR a list of 'leaf' elements if isinstance(y_leafs, list): for leaf in y_leafs: - attr = on_leaf(leaf) - ret_attrs_list.append(attr) + attr = on_leaf(leaf, is_leaf_list) + ret_attrs.append(attr) else: - attr = on_leaf(y_leafs) - ret_attrs_list.append(attr) + attr = on_leaf(y_leafs, is_leaf_list) + ret_attrs.append(attr) - return ret_attrs_list + return ret_attrs -def on_leaf(leaf: OrderedDict) -> dict: +def on_leaf(leaf: OrderedDict, is_leaf_list: bool) -> dict: """ Parse a single 'leaf' element Args: @@ -180,14 +264,13 @@ def on_leaf(leaf: OrderedDict) -> dict: Returns: dictionary - parsed 'leaf' element """ + mandatory = False if leaf.get('mandatory') is not None: - mandatory = leaf.get('mandatory').get('@value') + mandatory = True attr = { 'name': leaf.get('@name'), - 'is-leaf-list': leaf.get('__isleafList'), + 'description': get_description(leaf), + 'is-leaf-list': is_leaf_list, 'mandatory': mandatory } - return attr - -def on_list_container(cont): - pass \ No newline at end of file + return attr \ No newline at end of file From 73a3b121a428f83245d1c236bdf4b38e0ef1c504 Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Wed, 12 May 2021 11:31:47 +0000 Subject: [PATCH 081/173] Done parser for 'grouping' BUT need to deeply test it, added functions - get_leafs(), get_leaf_lists(), get_choices(), get_uses_grouping() Signed-off-by: Vadym Hlushko --- sonic_cli_gen/generator.py | 1 + sonic_cli_gen/yang_parser.py | 198 ++++++++++++++++++++++++----------- 2 files changed, 137 insertions(+), 62 deletions(-) diff --git a/sonic_cli_gen/generator.py b/sonic_cli_gen/generator.py index c328fb9995..6a3b6f2283 100644 --- a/sonic_cli_gen/generator.py +++ b/sonic_cli_gen/generator.py @@ -21,6 +21,7 @@ def generate_config_plugin(self): """ Generate CLI plugin for 'config' CLI group. """ parser = YangParser(self.yang_model_name) yang_dict = parser.parse_yang_model() + import pprint; pprint.pprint(yang_dict) pass def generate_show_plugin(self): diff --git a/sonic_cli_gen/yang_parser.py b/sonic_cli_gen/yang_parser.py index 5e1229e103..a7de4885d8 100644 --- a/sonic_cli_gen/yang_parser.py +++ b/sonic_cli_gen/yang_parser.py @@ -12,7 +12,7 @@ class YangParser: Attributes: yang_model_name: Name of the YANG model file conf_mgmt: Instance of Config Mgmt class to help parse YANG models - idx_yJson: Index of YANG model file (1 attr) inside conf_mgmt.sy.yJson object + idx_yJson: Index of yang_model_file (1 attr) inside conf_mgmt.sy.yJson object y_module: Reference to 'module' entity from YANG model file y_top_level_container: Reference to top level 'container' entity from YANG model file y_table_containers: Reference to 'container' entities from YANG model file @@ -81,33 +81,22 @@ def parse_yang_model(self) -> dict: # 'table' container it is a container that goes after 'top level' container if isinstance(self.y_table_containers, list): for tbl_cont in self.y_table_containers: - y2d_elem = on_table_container(tbl_cont) + y2d_elem = on_table_container(self.y_module, tbl_cont) self.yang_2_dict['tables'].append(y2d_elem) else: - y2d_elem = on_table_container(self.y_table_containers) + y2d_elem = on_table_container(self.y_module, self.y_table_containers) self.yang_2_dict['tables'].append(y2d_elem) - - return self.yang_2_dict -def get_description(y_entity: OrderedDict) -> str: - """ Parse 'description' entity from any YANG element - - Args: - y_entity: reference to YANG 'container' OR 'list' OR 'leaf' ... - Returns: - str - text of the 'description' - """ + return self.yang_2_dict - if y_entity.get('description') is not None: - return y_entity.get('description').get('text') - else: - return '' +#------------------------------HANDLERS--------------------------------# -def on_table_container(tbl_cont: OrderedDict) -> dict: +def on_table_container(y_module: OrderedDict, tbl_cont: OrderedDict) -> dict: """ Parse 'table' container, 'table' container goes after 'top level' container Args: + y_module: reference to 'module' tbl_cont: reference to 'table' container Returns: dictionary - element for self.yang_2_dict['tables'] @@ -120,39 +109,45 @@ def on_table_container(tbl_cont: OrderedDict) -> dict: 'static-objects': list() } - # determine if 'container' have a 'list' entity - tbl_cont_lists = tbl_cont.get('list') - - if tbl_cont_lists is None: - is_list = False + # determine if 'table container' have a 'list' entity + if tbl_cont.get('list') is None: # 'object' container goes after 'table' container # 'object' container have 2 types - list (like sonic-flex_counter.yang) and NOT list (like sonic-device_metadata.yang) obj_cont = tbl_cont.get('container') if isinstance(obj_cont, list): for cont in obj_cont: - static_obj_elem = on_container(cont, is_list) + static_obj_elem = on_object_container(y_module, cont, is_list=False) y2d_elem['static-objects'].append(static_obj_elem) else: - static_obj_elem = on_container(obj_cont, is_list) + static_obj_elem = on_object_container(y_module, obj_cont, is_list=False) y2d_elem['static-objects'].append(static_obj_elem) else: - is_list = True + tbl_cont_lists = tbl_cont.get('list') # 'container' can have more than 1 'list' if isinstance(tbl_cont_lists, list): for _list in tbl_cont_lists: - dynamic_obj_elem = on_container(_list, is_list) + dynamic_obj_elem = on_object_container(y_module, _list, is_list=True) y2d_elem['dynamic-objects'].append(dynamic_obj_elem) else: - dynamic_obj_elem = on_container(tbl_cont_lists, is_list) + dynamic_obj_elem = on_object_container(y_module, tbl_cont_lists, is_list=True) y2d_elem['dynamic-objects'].append(dynamic_obj_elem) return y2d_elem -def on_container(cont: OrderedDict, is_list: bool) -> dict: - """ Parse a 'container' that have only 'leafs' or 'list' with 'leafs' +def on_object_container(y_module: OrderedDict, cont: OrderedDict, is_list: bool) -> dict: + """ Parse a 'object container'. + 'Object container' represent OBJECT inside Config DB schema: + { + "TABLE": { + "OBJECT": { + "attr": "value" + } + } + } Args: - cont: reference to 'container' + y_module: reference to 'module' + cont: reference to 'object container' Returns: dictionary - element for y2d_elem['static-objects'] OR y2d_elem['dynamic-objects'] """ @@ -164,30 +159,63 @@ def on_container(cont: OrderedDict, is_list: bool) -> dict: } if is_list: - obj_elem['key'] = cont.get('key').get('@value') + obj_elem['keys'] = get_list_keys(cont) attrs_list = list() + attrs_list.extend(get_leafs(cont)) + attrs_list.extend(get_leaf_lists(cont)) + attrs_list.extend(get_choices(y_module, cont)) + # TODO: need to test 'grouping' + #attrs_list.extend(get_uses_grouping(y_module, cont)) - if cont.get('leaf') is not None: - is_leaf_list = False - ret_leafs = on_leafs(cont.get('leaf'), is_leaf_list) - attrs_list.extend(ret_leafs) + obj_elem['attrs'] = attrs_list - if cont.get('leaf-list') is not None: - is_leaf_list = True - ret_leaf_lists = on_leafs(cont.get('leaf-list'), is_leaf_list) - attrs_list.extend(ret_leaf_lists) + return obj_elem - if cont.get('choice') is not None: - y_choices = cont.get('choice') - ret_choice_leafs = on_choices(y_choices) - attrs_list.extend(ret_choice_leafs) +def on_grouping(y_module: OrderedDict, y_grouping, y_uses) -> list: + """ Parse a YANG 'grouping' and 'uses' entities + 'grouping' element can have - 'leaf', 'leaf-list', 'choice' - obj_elem['attrs'] = attrs_list + Args: + y_module: reference to 'module' + y_grouping: reference to 'grouping' + y_uses: reference to 'uses' + Returns: + dictionary - element for obj_elem['attrs'], 'attrs' contain a parsed 'leafs' + """ - return obj_elem + ret_attrs = list() + + if isinstance(y_uses, list): + if isinstance(y_grouping, list): + for use in y_uses: + for group in y_grouping: + if use.get('@name') == group.get('@name'): + ret_attrs.extend(get_leafs(group)) + ret_attrs.extend(get_leaf_lists(group)) + ret_attrs.extend(get_choices(y_module, group)) + else: + for use in y_uses: + if use.get('@name') == y_grouping.get('@name'): + ret_attrs.extend(get_leafs(y_grouping)) + ret_attrs.extend(get_leaf_lists(y_grouping)) + ret_attrs.extend(get_choices(y_module, y_grouping)) + else: + if isinstance(y_grouping, list): + for group in y_grouping: + if y_uses.get('@name') == group.get('@name'): + ret_attrs.extend(get_leafs(group)) + ret_attrs.extend(get_leaf_lists(group)) + ret_attrs.extend(get_choices(y_module, group)) + else: + if y_uses.get('@name') == y_grouping.get('@name'): + ret_attrs.extend(get_leafs(y_grouping)) + ret_attrs.extend(get_leaf_lists(y_grouping)) + ret_attrs.extend(get_choices(y_module, y_grouping)) + + return ret_attrs -def on_choices(y_choices) -> list: +def on_choices(y_module: OrderedDict, y_choices) -> list: """ Parse a YANG 'choice' entities Args: @@ -201,18 +229,20 @@ def on_choices(y_choices) -> list: # the YANG model can have multiple 'choice' entities inside 'container' or 'list' if isinstance(y_choices, list): for choice in y_choices: - attrs = on_choice_cases(choice.get('case')) + attrs = on_choice_cases(y_module, choice.get('case')) ret_attrs.extend(attrs) else: - ret_attrs = on_choice_cases(y_choices.get('case')) + ret_attrs = on_choice_cases(y_module, y_choices.get('case')) return ret_attrs -def on_choice_cases(y_cases: list) -> list: +def on_choice_cases(y_module: OrderedDict, y_cases: list) -> list: """ Parse a single YANG 'case' entity from 'choice' entity + 'case' element can have inside - 'leaf', 'leaf-list', 'uses' Args: - cont: reference to 'case' + y_module: reference to 'module' + y_cases: reference to 'case' Returns: dictionary - element for obj_elem['attrs'], 'attrs' contain a parsed 'leafs' """ @@ -221,15 +251,10 @@ def on_choice_cases(y_cases: list) -> list: if isinstance(y_cases, list): for case in y_cases: - if case.get('leaf') is not None: - is_leaf_list = False - ret_leafs = on_leafs(case.get('leaf'), is_leaf_list) - ret_attrs.extend(ret_leafs) - - if case.get('leaf-list') is not None: - is_leaf_list = True - ret_leaf_lists = on_leafs(case.get('leaf-list'), is_leaf_list) - ret_attrs.extend(ret_leaf_lists) + ret_attrs.extend(get_leafs(case)) + ret_attrs.extend(get_leaf_lists(case)) + # TODO: need to deeply test it + #ret_attrs.extend(get_uses_grouping(y_module, case)) else: raise Exception('It has no sense to using a single "case" element inside "choice" element') @@ -272,5 +297,54 @@ def on_leaf(leaf: OrderedDict, is_leaf_list: bool) -> dict: attr = { 'name': leaf.get('@name'), 'description': get_description(leaf), 'is-leaf-list': is_leaf_list, - 'mandatory': mandatory } - return attr \ No newline at end of file + 'is-mandatory': mandatory } + return attr + +#----------------------GETERS-------------------------# + +def get_description(y_entity: OrderedDict) -> str: + """ Parse 'description' entity from any YANG element + + Args: + y_entity: reference to YANG 'container' OR 'list' OR 'leaf' ... + Returns: + str - text of the 'description' + """ + + if y_entity.get('description') is not None: + return y_entity.get('description').get('text') + else: + return '' + +def get_leafs(y_entity: OrderedDict) -> list: + if y_entity.get('leaf') is not None: + return on_leafs(y_entity.get('leaf'), is_leaf_list=False) + + return [] + +def get_leaf_lists(y_entity: OrderedDict) -> list: + if y_entity.get('leaf-list') is not None: + return on_leafs(y_entity.get('leaf-list'), is_leaf_list=True) + + return [] + +def get_choices(y_module: OrderedDict, y_entity: OrderedDict) -> list: + if y_entity.get('choice') is not None: + return on_choices(y_module, y_entity.get('choice')) + + return [] + +def get_uses_grouping(y_module: OrderedDict, y_entity: OrderedDict) -> list: + if y_entity.get('uses') is not None and y_module.get('grouping') is not None: + return on_grouping(y_module, y_module.get('grouping'), y_entity.get('uses')) + + return [] + +def get_list_keys(y_list: OrderedDict) -> list: + ret_list = list() + keys = y_list.get('key').get('@value').split() + for k in keys: + key = { 'name': k } + ret_list.append(key) + + return ret_list \ No newline at end of file From 718f8abbb5611556534403e00ed9766d2bdb4fe6 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Wed, 12 May 2021 14:37:59 +0300 Subject: [PATCH 082/173] align with implementation Signed-off-by: Stepan Blyschak --- .../templates/sonic-cli-gen/config.py.j2 | 19 ++++++++++--------- .../templates/sonic-cli-gen/show.py.j2 | 14 +++++++------- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 b/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 index 4d70f9d1e8..10fca2512f 100644 --- a/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 +++ b/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 @@ -122,7 +122,7 @@ def clear_list_entry_validated(db, table, key, attr): {%- for arg in args %} @click.argument( "{{ cli_name(arg.name) }}", - nargs={% if arg.is_list %}-1{% else %}1{% endif %}, + nargs={% if arg["is-leaf-list"] %}-1{% else %}1{% endif %}, required=True, ) {%- endfor %} @@ -132,6 +132,7 @@ def clear_list_entry_validated(db, table, key, attr): {%- for opt in opts %} @click.option( "--{{ cli_name(opt.name) }}", + help="{{ opt.description }}{% if opt.['is-mandatory'] %}[mandatory]{% endif %}, ) {%- endfor %} {%- endmacro %} @@ -237,7 +238,7 @@ def {{ list_update_group }}_delete( {% macro config_object_list_update_all(group, table, object) %} {% for attr in object.attrs %} -{% if attr.is_list %} +{% if attr["is-leaf-list"] %} {{ config_object_list_update(group, table, object, attr) }} {% endif %} {% endfor %} @@ -314,7 +315,7 @@ def {{ group }}_add(db, {{ pythonize(object["keys"] + object.attrs) }}): data = {} {%- for attr in object.attrs %} if {{ pythonize([attr]) }} is not None: -{%- if not attr.is_list %} +{%- if not attr["is-leaf-list"] %} data["{{ attr.name }}"] = {{ pythonize([attr]) }} {%- else %} data["{{ attr.name }}"] = {{ pythonize([attr]) }}.split(",") @@ -350,7 +351,7 @@ def {{ group }}_update(db, {{ pythonize(object["keys"] + object.attrs) }}): data = {} {%- for attr in object.attrs %} if {{ pythonize([attr]) }} is not None: -{%- if not attr.is_list %} +{%- if not attr["is-leaf-list"] %} data["{{ attr.name }}"] = {{ pythonize([attr]) }} {%- else %} data["{{ attr.name }}"] = {{ pythonize([attr]) }}.split(",") @@ -387,7 +388,7 @@ def {{ group }}_delete(db, {{ pythonize(object["keys"]) }}): {% macro config_dynamic_object(table, object) %} {# Generate another nesting group in case table holds two types of objects #} -{% if table.dynamic_objects|length > 1 %} +{% if table["dynamic-objects"]|length > 1 %} {% set group = table.name + "_" + object.name %} @{{ table.name }}.group(name="{{ cli_name(object.name) }}", cls=clicommon.AliasedGroup) @@ -414,12 +415,12 @@ def {{ table.name }}(): pass -{% if "static_objects" in table %} -{% for object in table.static_objects %} +{% if "static-objects" in table %} +{% for object in table["static-objects"] %} {{ config_static_object(table, object) }} {% endfor %} -{% elif "dynamic_objects" in table %} -{% for object in table.dynamic_objects %} +{% elif "dynamic-objects" in table %} +{% for object in table["dynamic-objects"] %} {{ config_dynamic_object(table, object) }} {% endfor %} {% endif %} diff --git a/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 b/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 index 191a466852..98b06e1e86 100644 --- a/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 +++ b/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 @@ -7,7 +7,7 @@ import natsort import utilities_common.cli as clicommon {% macro print_attr(attr) %} -{%- if not attr.is_list %} +{%- if not attr["is-leaf-list"] %} entry.get("{{ attr.name }}", "N/A") {%- else %} "\n".join(entry.get("{{ attr.name }}", [])) @@ -23,7 +23,7 @@ entry.get("{{ attr.name }}", "N/A") {% for table in tables %} -{% if "static_objects" in table %} +{% if "static-objects" in table %} @click.group(name="{{ cli_name(table.name) }}", cls=clicommon.AliasedGroup) def {{ table.name }}(): @@ -31,7 +31,7 @@ def {{ table.name }}(): pass -{% for object in table.static_objects %} +{% for object in table["static-objects"] %} @{{ table.name }}.command(name="{{ cli_name(object.name) }}") @clicommon.pass_db def {{ table.name }}_{{ object.name }}(db): @@ -47,8 +47,8 @@ def {{ table.name }}_{{ object.name }}(db): click.echo(tabulate.tabulate(body, header)) {% endfor %} -{% elif "dynamic_objects" in table %} -{% if table.dynamic_objects|length > 1 %} +{% elif "dynamic-objects" in table %} +{% if table["dynamic-objects"]|length > 1 %} @click.group(name="{{ cli_name(table.name) }}", cls=clicommon.AliasedGroup) def {{ table.name }}(): @@ -56,9 +56,9 @@ def {{ table.name }}(): pass {% endif %} -{% for object in table.dynamic_objects %} +{% for object in table["dynamic-objects"] %} {# Generate another nesting group in case table holds two types of objects #} -{% if table.dynamic_objects|length > 1 %} +{% if table["dynamic-objects"]|length > 1 %} {% set group = table.name %} {% set name = object.name %} {% else %} From fa6299c55044a272a415308233b7633a88552412 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Wed, 12 May 2021 14:46:00 +0300 Subject: [PATCH 083/173] generate cli from templates Signed-off-by: Stepan Blyschak --- sonic_cli_gen/generator.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/sonic_cli_gen/generator.py b/sonic_cli_gen/generator.py index 6a3b6f2283..373fd55774 100644 --- a/sonic_cli_gen/generator.py +++ b/sonic_cli_gen/generator.py @@ -1,9 +1,8 @@ #!/usr/bin/env python -try: - from sonic_cli_gen.yang_parser import YangParser -except ImportError as e: - raise ImportError("%s - required module not found" % str(e)) +import jinja2 + +from sonic_cli_gen.yang_parser import YangParser class CliGenerator: """ SONiC CLI generator. This class provides public API @@ -16,23 +15,30 @@ def __init__(self, """ Initialize PackageManager. """ self.yang_model_name = yang_model + self.loader = jinja2.FileSystemLoader(['/usr/share/sonic/templates/sonic-cli-gen/']) + self.env = jinja2.Environment(loader=self.loader) def generate_config_plugin(self): """ Generate CLI plugin for 'config' CLI group. """ parser = YangParser(self.yang_model_name) yang_dict = parser.parse_yang_model() - import pprint; pprint.pprint(yang_dict) - pass + template = self.env.get_template('config.py.j2') + with open('config.py', 'w') as config_py: + config_py.write(template.render(yang_dict)) + def generate_show_plugin(self): """ Generate CLI plugin for 'show' CLI group. """ parser = YangParser(self.yang_model_name) yang_dict = parser.parse_yang_model() - pass + template = self.env.get_template('show.py.j2') + with open('show.py', 'w') as show_py: + show_py.write(template.render(yang_dict)) # to be implemented in the next Phases def generate_sonic_clear_plugin(self): """ Generate CLI plugin for 'sonic-clear' CLI group. """ parser = YangParser(self.yang_model_name) yang_dict = parser.parse_yang_model() - pass + raise NotImplementedError + From 9d9f66ba5cb7aa35468754d49e6fb3ea00d405fa Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Wed, 12 May 2021 12:14:59 +0000 Subject: [PATCH 084/173] Table can only have 'static-objects' OR 'dynamic-objects' Signed-off-by: Vadym Hlushko --- sonic_cli_gen/yang_parser.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/sonic_cli_gen/yang_parser.py b/sonic_cli_gen/yang_parser.py index a7de4885d8..5556727924 100644 --- a/sonic_cli_gen/yang_parser.py +++ b/sonic_cli_gen/yang_parser.py @@ -104,13 +104,14 @@ def on_table_container(y_module: OrderedDict, tbl_cont: OrderedDict) -> dict: y2d_elem = { 'name': tbl_cont.get('@name'), - 'description': get_description(tbl_cont), - 'dynamic-objects': list(), - 'static-objects': list() + 'description': get_description(tbl_cont) } # determine if 'table container' have a 'list' entity if tbl_cont.get('list') is None: + # TODO: comment aboit it + y2d_elem['static-objects'] = list() + # 'object' container goes after 'table' container # 'object' container have 2 types - list (like sonic-flex_counter.yang) and NOT list (like sonic-device_metadata.yang) obj_cont = tbl_cont.get('container') @@ -122,6 +123,10 @@ def on_table_container(y_module: OrderedDict, tbl_cont: OrderedDict) -> dict: static_obj_elem = on_object_container(y_module, obj_cont, is_list=False) y2d_elem['static-objects'].append(static_obj_elem) else: + + # TODO: comment aboit it + y2d_elem['dynamic-objects'] = list() + tbl_cont_lists = tbl_cont.get('list') # 'container' can have more than 1 'list' if isinstance(tbl_cont_lists, list): From 38f4365881d2e4be523051aa8c837160566042a4 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Wed, 12 May 2021 15:33:14 +0300 Subject: [PATCH 085/173] add missing " in help for option Signed-off-by: Stepan Blyschak --- sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 b/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 index 10fca2512f..1ad0341656 100644 --- a/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 +++ b/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 @@ -132,7 +132,7 @@ def clear_list_entry_validated(db, table, key, attr): {%- for opt in opts %} @click.option( "--{{ cli_name(opt.name) }}", - help="{{ opt.description }}{% if opt.['is-mandatory'] %}[mandatory]{% endif %}, + help="{{ opt.description }}{% if opt['is-mandatory'] %}[mandatory]{% endif %}", ) {%- endfor %} {%- endmacro %} @@ -419,7 +419,8 @@ def {{ table.name }}(): {% for object in table["static-objects"] %} {{ config_static_object(table, object) }} {% endfor %} -{% elif "dynamic-objects" in table %} + +{% if "dynamic-objects" in table %} {% for object in table["dynamic-objects"] %} {{ config_dynamic_object(table, object) }} {% endfor %} From 0826ac5bd835e74842e1310f1bf56584350f757d Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Wed, 12 May 2021 12:56:32 +0000 Subject: [PATCH 086/173] Added remove_keys(), get_mandatory() Signed-off-by: Vadym Hlushko --- sonic_cli_gen/yang_parser.py | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/sonic_cli_gen/yang_parser.py b/sonic_cli_gen/yang_parser.py index 5556727924..89a9822689 100644 --- a/sonic_cli_gen/yang_parser.py +++ b/sonic_cli_gen/yang_parser.py @@ -137,8 +137,21 @@ def on_table_container(y_module: OrderedDict, tbl_cont: OrderedDict) -> dict: dynamic_obj_elem = on_object_container(y_module, tbl_cont_lists, is_list=True) y2d_elem['dynamic-objects'].append(dynamic_obj_elem) + # remove 'keys' elementsfrom 'attrs' list + remove_keys(y2d_elem['dynamic-objects']) + return y2d_elem +# TODO: think about name +def remove_keys(dynamic_objects: OrderedDict): + for obj in dynamic_objects: + for key in obj.get('keys'): + for attr in obj.get('attrs'): + if key.get('name') == attr.get('name'): + key['description'] = attr.get('description') + obj['attrs'].remove(attr) + break + def on_object_container(y_module: OrderedDict, cont: OrderedDict, is_list: bool) -> dict: """ Parse a 'object container'. 'Object container' represent OBJECT inside Config DB schema: @@ -295,18 +308,21 @@ def on_leaf(leaf: OrderedDict, is_leaf_list: bool) -> dict: dictionary - parsed 'leaf' element """ - mandatory = False - if leaf.get('mandatory') is not None: - mandatory = True - attr = { 'name': leaf.get('@name'), 'description': get_description(leaf), 'is-leaf-list': is_leaf_list, - 'is-mandatory': mandatory } + 'is-mandatory': get_mandatory(leaf) } + return attr #----------------------GETERS-------------------------# +def get_mandatory(y_leaf: OrderedDict) -> bool: + if y_leaf.get('mandatory') is not None: + return True + + return False + def get_description(y_entity: OrderedDict) -> str: """ Parse 'description' entity from any YANG element From 069000f84a2a5b3cac8df0d013e9510ebbf42b62 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Wed, 12 May 2021 16:56:39 +0300 Subject: [PATCH 087/173] fix issues Signed-off-by: Stepan Blyschak --- .../templates/sonic-cli-gen/config.py.j2 | 10 +++++++++- .../templates/sonic-cli-gen/show.py.j2 | 5 ++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 b/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 index 1ad0341656..ac19835cd1 100644 --- a/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 +++ b/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 @@ -2,8 +2,13 @@ """ Autogenerated config CLI plugin """ import click -from config import config_mgmt import utilities_common.cli as clicommon +import utilities_common.general as general +from config import config_mgmt + + +# Load sonic-cfggen from source since /usr/local/bin/sonic-cfggen does not have .py extension. +sonic_cfggen = general.load_module_from_source('sonic_cfggen', '/usr/local/bin/sonic-cfggen') def exit_cli(*args, **kwargs): @@ -17,6 +22,7 @@ def validate_config_or_raise(cfg): """ Validate config db data using ConfigMgmt """ try: + cfg = sonic_cfggen.FormatConverter.to_serialized(cfg) config_mgmt.ConfigMgmt().loadData(cfg) except Exception as err: raise Exception('Failed to validate configuration: {}'.format(err)) @@ -419,12 +425,14 @@ def {{ table.name }}(): {% for object in table["static-objects"] %} {{ config_static_object(table, object) }} {% endfor %} +{% endif %} {% if "dynamic-objects" in table %} {% for object in table["dynamic-objects"] %} {{ config_dynamic_object(table, object) }} {% endfor %} {% endif %} + {% endfor %} def register(cli): diff --git a/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 b/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 index 98b06e1e86..58e7dfd2e0 100644 --- a/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 +++ b/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 @@ -41,7 +41,7 @@ def {{ table.name }}_{{ object.name }}(db): body = [] table = db.cfgdb.get_table("{{ table.name }}") - entry = table.get("{{ object.name }}") + entry = table.get("{{ object.name }}", {}) row = [{%- for attr in object.attrs -%} {{ print_attr(attr) }}, {%- endfor %}] body.append(row) click.echo(tabulate.tabulate(body, header)) @@ -77,8 +77,7 @@ def {{ name }}(db): body = [] table = db.cfgdb.get_table("{{ table.name }}") - for key in natsort.natsorted(table): - entry = table[key] + for key, entry in natsort.natsorted(table).items(): if not isinstance(key, tuple): key = (key,) From 0ba3cf40e3bbd7bb8ad36fdedfb5be2567631dc0 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Thu, 13 May 2021 13:48:11 +0300 Subject: [PATCH 088/173] improve manifest types validation Signed-off-by: Stepan Blyschak --- sonic_package_manager/manifest.py | 16 ++++++++++------ tests/sonic_package_manager/conftest.py | 2 +- tests/sonic_package_manager/test_manifest.py | 14 ++++++++++++++ 3 files changed, 25 insertions(+), 7 deletions(-) diff --git a/sonic_package_manager/manifest.py b/sonic_package_manager/manifest.py index d2b9dab818..c126e2eef1 100644 --- a/sonic_package_manager/manifest.py +++ b/sonic_package_manager/manifest.py @@ -92,8 +92,10 @@ class ManifestRoot(ManifestNode): def marshal(self, value: Optional[dict]): result = {} - if value is None: - value = {} + value = value or {} + + if not isinstance(value, dict): + raise ManifestError(f'"{self.key}" field has to be a dictionary') for item in self.items: next_value = value.get(item.key) @@ -115,7 +117,7 @@ def marshal(self, value): if value is None: if self.default is not None: return self.default - raise ManifestError(f'{self.key} is a required field but it is missing') + raise ManifestError(f'"{self.key}" is a required field but it is missing') try: return_value = self.type.marshal(value) except Exception as err: @@ -130,10 +132,12 @@ class ManifestArray(ManifestNode): type: Any def marshal(self, value): - if value is None: - return [] - return_value = [] + value = value or [] + + if not isinstance(value, list): + raise ManifestError(f'"{self.key}" has to be of type list') + try: for item in value: return_value.append(self.type.marshal(item)) diff --git a/tests/sonic_package_manager/conftest.py b/tests/sonic_package_manager/conftest.py index 9d5fc01b59..2788a75cd3 100644 --- a/tests/sonic_package_manager/conftest.py +++ b/tests/sonic_package_manager/conftest.py @@ -154,7 +154,7 @@ def add(self, repo, reference, name, version, components=None, 'warm-shutdown': warm_shutdown or {}, 'fast-shutdown': fast_shutdown or {}, }, - 'processes': processes or {} + 'processes': processes or [], }, 'components': components or {}, } diff --git a/tests/sonic_package_manager/test_manifest.py b/tests/sonic_package_manager/test_manifest.py index efdcc558ab..2f201b8107 100644 --- a/tests/sonic_package_manager/test_manifest.py +++ b/tests/sonic_package_manager/test_manifest.py @@ -57,6 +57,20 @@ def test_manifest_v1_mounts_invalid(): 'mounts': [{'not-source': 'a', 'target': 'b', 'type': 'bind'}]}}) +def test_manifest_invalid_root_type(): + manifest_json_input = {'package': { 'name': 'test', 'version': '1.0.0'}, + 'service': {'name': 'test'}, 'container': 'abc'} + with pytest.raises(ManifestError): + Manifest.marshal(manifest_json_input) + + +def test_manifest_invalid_array_type(): + manifest_json_input = {'package': { 'name': 'test', 'version': '1.0.0'}, + 'service': {'name': 'test', 'warm-shutdown': {'after': 'bgp'}}} + with pytest.raises(ManifestError): + Manifest.marshal(manifest_json_input) + + def test_manifest_v1_unmarshal(): manifest_json_input = {'package': {'name': 'test', 'version': '1.0.0', 'depends': [ From 1723f0f29050a6ba4a8be4ce4ec9f6887dd60c8a Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Thu, 13 May 2021 12:33:15 +0000 Subject: [PATCH 089/173] Refactored generator.py: generate_cli_plugin(), get_cli_plugin_path() Signed-off-by: Vadym Hlushko --- sonic_cli_gen/generator.py | 38 +++++++++++++++--------------------- sonic_cli_gen/main.py | 15 ++++---------- sonic_cli_gen/yang_parser.py | 7 ++----- 3 files changed, 22 insertions(+), 38 deletions(-) diff --git a/sonic_cli_gen/generator.py b/sonic_cli_gen/generator.py index 373fd55774..0a02a104bc 100644 --- a/sonic_cli_gen/generator.py +++ b/sonic_cli_gen/generator.py @@ -1,6 +1,8 @@ #!/usr/bin/env python import jinja2 +import os +import pkgutil from sonic_cli_gen.yang_parser import YangParser @@ -18,27 +20,19 @@ def __init__(self, self.loader = jinja2.FileSystemLoader(['/usr/share/sonic/templates/sonic-cli-gen/']) self.env = jinja2.Environment(loader=self.loader) - def generate_config_plugin(self): - """ Generate CLI plugin for 'config' CLI group. """ + def generate_cli_plugin(self, cli_group, plugin_name): + """ Generate CLI plugin. """ parser = YangParser(self.yang_model_name) yang_dict = parser.parse_yang_model() - template = self.env.get_template('config.py.j2') - with open('config.py', 'w') as config_py: - config_py.write(template.render(yang_dict)) - - - def generate_show_plugin(self): - """ Generate CLI plugin for 'show' CLI group. """ - parser = YangParser(self.yang_model_name) - yang_dict = parser.parse_yang_model() - template = self.env.get_template('show.py.j2') - with open('show.py', 'w') as show_py: - show_py.write(template.render(yang_dict)) - - # to be implemented in the next Phases - def generate_sonic_clear_plugin(self): - """ Generate CLI plugin for 'sonic-clear' CLI group. """ - parser = YangParser(self.yang_model_name) - yang_dict = parser.parse_yang_model() - raise NotImplementedError - + plugin_path = get_cli_plugin_path(cli_group, plugin_name + '_yang.py') + template = self.env.get_template(cli_group + '.py.j2') + with open(plugin_path, 'w') as plugin_py: + plugin_py.write(template.render(yang_dict)) + +def get_cli_plugin_path(command, plugin_name): + pkg_loader = pkgutil.get_loader(f'{command}.plugins') + if pkg_loader is None: + raise PackageManagerError(f'Failed to get plugins path for {command} CLI') + plugins_pkg_path = os.path.dirname(pkg_loader.path) + + return os.path.join(plugins_pkg_path, plugin_name) \ No newline at end of file diff --git a/sonic_cli_gen/main.py b/sonic_cli_gen/main.py index bf783e8541..8cf748e394 100644 --- a/sonic_cli_gen/main.py +++ b/sonic_cli_gen/main.py @@ -1,12 +1,7 @@ #!/usr/bin/env python -try: - import sys - import os - import click - from sonic_cli_gen.generator import CliGenerator -except ImportError as e: - raise ImportError("%s - required module not found" % str(e)) +import click +from sonic_cli_gen.generator import CliGenerator @click.group() @click.pass_context @@ -20,8 +15,7 @@ def cli(ctx): def generate_config(ctx, yang_model_name): """ Generate CLI plugin (click) for 'config' CLI group. """ gen = CliGenerator(yang_model_name) - gen.generate_config_plugin() - pass + gen.generate_cli_plugin(cli_group='config', plugin_name=yang_model_name) @cli.command() @click.argument('yang_model_name') @@ -29,8 +23,7 @@ def generate_config(ctx, yang_model_name): def generate_show(ctx, yang_model_name): """ Generate CLI plugin (click) for 'show' CLI group. """ gen = CliGenerator(yang_model_name) - gen.generate_show_plugin() - pass + gen.generate_cli_plugin(cli_group='show', plugin_name=yang_model_name) if __name__ == '__main__': cli() \ No newline at end of file diff --git a/sonic_cli_gen/yang_parser.py b/sonic_cli_gen/yang_parser.py index 89a9822689..a67a3d06d5 100644 --- a/sonic_cli_gen/yang_parser.py +++ b/sonic_cli_gen/yang_parser.py @@ -1,10 +1,7 @@ #!/usr/bin/env python -try: - from collections import OrderedDict - from config.config_mgmt import ConfigMgmt -except ImportError as e: - raise ImportError("%s - required module not found" % str(e)) +from collections import OrderedDict +from config.config_mgmt import ConfigMgmt class YangParser: """ YANG model parser From 3f03594e0569a904b0b405e8045bffc00fe69b65 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Thu, 13 May 2021 15:55:05 +0300 Subject: [PATCH 090/173] remove redundant file Signed-off-by: Stepan Blyschak --- .../sonic-what-just-happened.yang | 79 ------------------- 1 file changed, 79 deletions(-) delete mode 100644 sonic_package_manager/sonic-what-just-happened.yang diff --git a/sonic_package_manager/sonic-what-just-happened.yang b/sonic_package_manager/sonic-what-just-happened.yang deleted file mode 100644 index 1d02926b6f..0000000000 --- a/sonic_package_manager/sonic-what-just-happened.yang +++ /dev/null @@ -1,79 +0,0 @@ -module sonic-what-just-happened { - - yang-version 1.1; - - namespace "http://github.com/Azure/sonic-what-just-happened"; - prefix what-just-happened; - - description "What-Just-Happened yang Module for SONiC OS"; - - revision 2020-05-01 { - description "First Revision"; - } - - container sonic-what-just-happened { - - container WJH_CHANNEL { - - description "What-Just-Happened channels configuration"; - - list WJH_CHANNEL_LIST { - - description "List of What-Just-Happened channels"; - - key "name"; - - leaf name { - type string { - pattern "l1|forwarding|buffer"; - } - } - - leaf "type" { - type string { - pattern "raw|aggregated|raw_and_aggregated"; - } - } - - leaf drop_category_list { - type string; - } - } - /* end of WJH_CHANNEL_LIST */ - } - /* end of WJH_CHANNEL container */ - - container WJH { - - description "What-Just-Happened global configuration"; - - container global { - - leaf mode { - type string { - pattern "debug"; - } - default "debug"; - } - - leaf nice_level { - type int { - range -20..19; - } - default 1; - } - - leaf pci_bandwidth { - type int { - range 0..100; - } - default 50; - } - - } - } - /* end of container WJH */ - } - /* end of container sonic-what-just-happened */ -} -/* end of module sonic-what-just-happened */ \ No newline at end of file From 1554f001fcc62dcbd5b6e148dd77b4b14a43a2e6 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Thu, 13 May 2021 16:49:05 +0300 Subject: [PATCH 091/173] fix docstrings Signed-off-by: Stepan Blyschak --- sonic_package_manager/service_creator/creator.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sonic_package_manager/service_creator/creator.py b/sonic_package_manager/service_creator/creator.py index 9b4081a82d..4c618eb7ea 100644 --- a/sonic_package_manager/service_creator/creator.py +++ b/sonic_package_manager/service_creator/creator.py @@ -440,7 +440,7 @@ def generate_service_reconciliation_file(self, package): (path: /etc/sonic/_reconcile). Args: - package: Package object to generate dump plugin script for. + package: Package object to generate service reconciliation file for. Returns: None """ @@ -456,7 +456,7 @@ def set_initial_config(self, package): This method updates but does not override existing entries in tables. Args: - package: Package object to generate dump plugin script for. + package: Package object to set initial configuration for. Returns: None """ @@ -485,7 +485,7 @@ def remove_config(self, package): TODO: remove config from tables known to yang model Args: - package: Package object to generate dump plugin script for. + package: Package object remove initial configuration for. Returns: None """ From dccff0b07df9ce771ee7d41771182992f1f84e08 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Thu, 13 May 2021 19:06:24 +0300 Subject: [PATCH 092/173] use swsscommon.ConfigDBConnector Signed-off-by: Stepan Blyschak --- .../service_creator/creator.py | 145 ++++++++---------- .../service_creator/feature.py | 82 +++++----- .../service_creator/sonic_db.py | 140 +++++++++++------ tests/sonic_package_manager/conftest.py | 2 +- .../test_service_creator.py | 118 +++++++------- 5 files changed, 267 insertions(+), 220 deletions(-) diff --git a/sonic_package_manager/service_creator/creator.py b/sonic_package_manager/service_creator/creator.py index 64cb52df04..4125bf8802 100644 --- a/sonic_package_manager/service_creator/creator.py +++ b/sonic_package_manager/service_creator/creator.py @@ -5,23 +5,24 @@ import stat import subprocess from collections import defaultdict -from typing import Dict +from typing import Dict, Type import jinja2 as jinja2 from config.config_mgmt import ConfigMgmt from prettyprinter import pformat from toposort import toposort_flatten, CircularDependencyError +from utilities_common.general import load_module_from_source from sonic_package_manager.logger import log from sonic_package_manager.package import Package from sonic_package_manager.service_creator import ETC_SONIC_PATH from sonic_package_manager.service_creator.feature import FeatureRegistry -from sonic_package_manager.service_creator.sonic_db import ( - CONFIG_DB_JSON, - INIT_CFG_JSON -) +from sonic_package_manager.service_creator.sonic_db import SonicDB from sonic_package_manager.service_creator.utils import in_chroot +# Load sonic-cfggen from source since /usr/local/bin/sonic-cfggen does not have .py extension. +sonic_cfggen = load_module_from_source('sonic_cfggen', '/usr/local/bin/sonic-cfggen') + SERVICE_FILE_TEMPLATE = 'sonic.service.j2' TIMER_UNIT_TEMPLATE = 'timer.unit.j2' @@ -83,6 +84,16 @@ def set_executable_bit(filepath): os.chmod(filepath, st.st_mode | stat.S_IEXEC) +def remove_if_exists(path): + """ Remove filepath if it exists """ + + if not os.path.exists(path): + return + + os.remove(path) + log.info(f'removed {path}') + + def run_command(command: str): """ Run arbitrary bash command. Args: @@ -109,13 +120,13 @@ class ServiceCreator: def __init__(self, feature_registry: FeatureRegistry, - sonic_db, + sonic_db: Type[SonicDB], cfg_mgmt: ConfigMgmt): """ Initialize ServiceCreator with: Args: feature_registry: FeatureRegistry object. - sonic_db: SonicDb interface. + sonic_db: SonicDB interface. cfg_mgmt: ConfigMgmt instance. """ @@ -148,13 +159,11 @@ def create(self, self.generate_dump_script(package) self.generate_service_reconciliation_file(package) self.install_yang_module(package) - self.set_initial_config(package) self._post_operation_hook() if register_feature: - self.feature_registry.register(package.manifest, - state, owner) + self.feature_registry.register(package.manifest, state, owner) except (Exception, KeyboardInterrupt): self.remove(package, register_feature) raise @@ -175,26 +184,18 @@ def remove(self, """ name = package.manifest['service']['name'] - - def remove_file(path): - if os.path.exists(path): - os.remove(path) - log.info(f'removed {path}') - - remove_file(os.path.join(SYSTEMD_LOCATION, f'{name}.service')) - remove_file(os.path.join(SYSTEMD_LOCATION, f'{name}@.service')) - remove_file(os.path.join(SERVICE_MGMT_SCRIPT_LOCATION, f'{name}.sh')) - remove_file(os.path.join(DOCKER_CTL_SCRIPT_LOCATION, f'{name}.sh')) - remove_file(os.path.join(DEBUG_DUMP_SCRIPT_LOCATION, f'{name}')) - remove_file(os.path.join(ETC_SONIC_PATH, f'{name}_reconcile')) - + remove_if_exists(os.path.join(SYSTEMD_LOCATION, f'{name}.service')) + remove_if_exists(os.path.join(SYSTEMD_LOCATION, f'{name}@.service')) + remove_if_exists(os.path.join(SERVICE_MGMT_SCRIPT_LOCATION, f'{name}.sh')) + remove_if_exists(os.path.join(DOCKER_CTL_SCRIPT_LOCATION, f'{name}.sh')) + remove_if_exists(os.path.join(DEBUG_DUMP_SCRIPT_LOCATION, f'{name}')) + remove_if_exists(os.path.join(ETC_SONIC_PATH, f'{name}_reconcile')) self.update_dependent_list_file(package, remove=True) if deregister_feature and not keep_config: self.remove_config(package) self.uninstall_yang_module(package) - self._post_operation_hook() if deregister_feature: @@ -318,8 +319,8 @@ def update_dependent_list_file(self, package: Package, remove=False): package: Package to update packages dependent of it. Returns: None. - """ + name = package.manifest['service']['name'] dependent_of = package.manifest['service']['dependent-of'] host_service = package.manifest['service']['host-service'] @@ -480,24 +481,13 @@ def set_initial_config(self, package): init_cfg = package.manifest['package']['init-cfg'] if not init_cfg: return - - for tablename, content in init_cfg.items(): - if not isinstance(content, dict): - continue - - tables = self._get_tables(tablename) - - for key in content: - for table in tables: - cfg = content[key] - exists, old_fvs = table.get(key) - if exists: - cfg.update(old_fvs) - fvs = list(cfg.items()) - table.set(key, fvs) - - if package.metadata.yang_module_text: - self.validate_configs() + + for conn in self.sonic_db.get_connectors(): + cfg = conn.get_config() + new_cfg = init_cfg.copy() + sonic_cfggen.deep_update(new_cfg, cfg) + self.validate_config(new_cfg) + conn.mod_config(new_cfg) def remove_config(self, package): """ Remove configuration based on package YANG module. @@ -516,57 +506,56 @@ def remove_config(self, package): if module['module'] != module_name: continue - tables = self._get_tables(tablename) - for table in tables: - for key in table.getKeys(): - table._del(key) + for conn in self.sonic_db.get_connectors(): + keys = conn.get_table(tablename).keys() + for key in keys: + conn.set_entry(tablename, key, None) - def validate_configs(self): - """ Validate configuration through YANG """ + def validate_config(self, config): + """ Validate configuration through YANG. + + Args: + config: Config DB data. + Returns: + None. + Raises: + Exception: if config does not pass YANG validation. + """ - log.debug('validating running configuration') - self.cfg_mgmt.readConfigDB() - self.cfg_mgmt.loadData(self.cfg_mgmt.configdbJsonIn) + config = sonic_cfggen.FormatConverter.to_serialized(config) + log.debug(f'validating configuration {pformat(config)}') + # This will raise exception if configuration is not valid. + self.cfg_mgmt.loadData(config) - log.debug('validating saved configuration') - self.cfg_mgmt.readConfigDBJson(CONFIG_DB_JSON) - self.cfg_mgmt.loadData(self.cfg_mgmt.configdbJsonIn) + def install_yang_module(self, package: Package): + """ Install package's yang module in the system. - log.debug('validating initial configuration') - self.cfg_mgmt.readConfigDBJson(INIT_CFG_JSON) - self.cfg_mgmt.loadData(self.cfg_mgmt.configdbJsonIn) + Args: + package: Package object. + Returns: + None + """ - def install_yang_module(self, package: Package): if not package.metadata.yang_module_text: return self.cfg_mgmt.add_module(package.metadata.yang_module_text) def uninstall_yang_module(self, package: Package): + """ Uninstall package's yang module in the system. + + Args: + package: Package object. + Returns: + None + """ + if not package.metadata.yang_module_text: return + module_name = self.cfg_mgmt.get_module_name(package.metadata.yang_module_text) self.cfg_mgmt.remove_module(module_name) - def _get_tables(self, table_name): - """ Return swsscommon Tables for all kinds of configuration DBs """ - - tables = [] - - running_table = self.sonic_db.running_table(table_name) - if running_table is not None: - tables.append(running_table) - - persistent_table = self.sonic_db.persistent_table(table_name) - if persistent_table is not None: - tables.append(persistent_table) - - initial_table = self.sonic_db.initial_table(table_name) - if initial_table is not None: - tables.append(initial_table) - - return tables - def _post_operation_hook(self): """ Common operations executed after service is created/removed. """ diff --git a/sonic_package_manager/service_creator/feature.py b/sonic_package_manager/service_creator/feature.py index 229531c9b5..7127847073 100644 --- a/sonic_package_manager/service_creator/feature.py +++ b/sonic_package_manager/service_creator/feature.py @@ -16,6 +16,14 @@ } +def is_enabled(cfg): + return cfg.get('state', 'disabled').lower() == 'enabled' + + +def is_multi_instance(cfg): + return str(cfg.get('has_per_asic_scope', 'False')).lower() == 'true' + + class FeatureRegistry: """ FeatureRegistry class provides an interface to register/de-register new feature persistently. """ @@ -27,51 +35,63 @@ def register(self, manifest: Manifest, state: str = 'disabled', owner: str = 'local'): + """ Register feature in CONFIG DBs. + + Args: + manifest: Feature's manifest. + state: Desired feature admin state. + owner: Owner of this feature (kube/local). + Returns: + None. + """ + name = manifest['service']['name'] - for table in self._get_tables(): - cfg_entries = self.get_default_feature_entries(state, owner) - non_cfg_entries = self.get_non_configurable_feature_entries(manifest) + db_connectors = self._sonic_db.get_connectors() + cfg_entries = self.get_default_feature_entries(state, owner) + non_cfg_entries = self.get_non_configurable_feature_entries(manifest) - exists, current_cfg = table.get(name) + for conn in db_connectors: + current_cfg = conn.get_entry(FEATURE, name) new_cfg = cfg_entries.copy() # Override configurable entries with CONFIG DB data. - new_cfg = {**new_cfg, **dict(current_cfg)} + new_cfg = {**new_cfg, **current_cfg} # Override CONFIG DB data with non configurable entries. new_cfg = {**new_cfg, **non_cfg_entries} - table.set(name, list(new_cfg.items())) + conn.set_entry(FEATURE, name, new_cfg) def deregister(self, name: str): - for table in self._get_tables(): - table._del(name) + """ Deregister feature by name. + + Args: + name: Name of the feature in CONFIG DB. + Returns: + None + """ + + db_connetors = self._sonic_db.get_connectors() + for conn in db_connetors: + conn.set_entry(FEATURE, name, None) def is_feature_enabled(self, name: str) -> bool: """ Returns whether the feature is current enabled or not. Accesses running CONFIG DB. If no running CONFIG_DB table is found in tables returns False. """ - running_db_table = self._sonic_db.running_table(FEATURE) - if running_db_table is None: + conn = self._sonic_db.get_running_db_connector() + if conn is None: return False - exists, cfg = running_db_table.get(name) - if not exists: - return False - cfg = dict(cfg) - return cfg.get('state').lower() == 'enabled' + cfg = conn.get_entry(name) + return is_enabled(cfg) def get_multi_instance_features(self): - res = [] - init_db_table = self._sonic_db.initial_table(FEATURE) - for feature in init_db_table.getKeys(): - exists, cfg = init_db_table.get(feature) - assert exists - cfg = dict(cfg) - asic_flag = str(cfg.get('has_per_asic_scope', 'False')) - if asic_flag.lower() == 'true': - res.append(feature) - return res + """ Returns a list of features which run in asic namespace. """ + + conn = self._sonic_db.get_initial_db_connector() + features = conn.get_table(FEATURE) + return [feature for feature, cfg in features if is_multi_instance(cfg)] @staticmethod def get_default_feature_entries(state=None, owner=None) -> Dict[str, str]: @@ -94,15 +114,3 @@ def get_non_configurable_feature_entries(manifest) -> Dict[str, str]: 'has_global_scope': str(manifest['service']['host-service']), 'has_timer': str(manifest['service']['delayed']), } - - def _get_tables(self): - tables = [] - running = self._sonic_db.running_table(FEATURE) - if running is not None: # it's Ok if there is no database container running - tables.append(running) - persistent = self._sonic_db.persistent_table(FEATURE) - if persistent is not None: # it's Ok if there is no config_db.json - tables.append(persistent) - tables.append(self._sonic_db.initial_table(FEATURE)) # init_cfg.json is must - - return tables diff --git a/sonic_package_manager/service_creator/sonic_db.py b/sonic_package_manager/service_creator/sonic_db.py index 23c166715e..dc81fe0e3f 100644 --- a/sonic_package_manager/service_creator/sonic_db.py +++ b/sonic_package_manager/service_creator/sonic_db.py @@ -6,6 +6,7 @@ from swsscommon import swsscommon +from utilities_common.general import load_module_from_source from sonic_package_manager.service_creator import ETC_SONIC_PATH from sonic_package_manager.service_creator.utils import in_chroot @@ -14,48 +15,76 @@ INIT_CFG_JSON = os.path.join(ETC_SONIC_PATH, 'init_cfg.json') -class FileDbTable: - """ swsscommon.Table adapter for persistent DBs. """ +# Load sonic-cfggen from source since /usr/local/bin/sonic-cfggen does not have .py extension. +sonic_cfggen = load_module_from_source('sonic_cfggen', '/usr/local/bin/sonic-cfggen') - def __init__(self, file, table): - self._file = file - self._table = table - def getKeys(self): - with open(self._file) as stream: - config = json.load(stream) - return config.get(self._table, {}).keys() - - def get(self, key): - with open(self._file) as stream: - config = json.load(stream) - - table = config.get(self._table, {}) - exists = key in table - fvs_dict = table.get(key, {}) - fvs = list(fvs_dict.items()) - return exists, fvs - - def set(self, key, fvs): - with open(self._file) as stream: - config = json.load(stream) - - table = config.setdefault(self._table, {}) - table.update({key: dict(fvs)}) +class PersistentConfigDbConnector: + """ swsscommon.ConfigDBConnector adapter for persistent DBs. """ - with open(self._file, 'w') as stream: - json.dump(config, stream, indent=4) + def __init__(self, filepath): + self._filepath = filepath - def _del(self, key): - with open(self._file) as stream: + def get_config(self): + with open(self._filepath) as stream: config = json.load(stream) + config = sonic_cfggen.FormatConverter.to_deserialized(config) + return config + + def get_entry(self, table, key): + table = table.upper() + table_data = self.get_table(table) + return table_data.get(key, {}) + + def get_table(self, table): + table = table.upper() + config = self.get_config() + return config.get(table, {}) + + def set_entry(self, table, key, data): + table = table.upper() + config = self.get_config() + if data is None: + self._del_key(config, table, key) + else: + table_data = config.setdefault(table, {}) + table_data[key] = data + self._write_config(config) + + def mod_entry(self, table, key, data): + table = table.upper() + config = self.get_config() + if data is None: + self._del_key(config, table, key) + else: + table_data = config.setdefault(table, {}) + curr_data = table_data.setdefault(key, {}) + curr_data.update(data) + self._write_config(config) + + def mod_config(self, config): + for table_name in config: + table_data = config[table_name] + if table_data is None: + self._del_table(config, table) + continue + for key in table_data: + self.mod_entry(table_name, key, table_data[key]) + + def _del_table(self, config, table): + with contextlib.suppress(KeyError): + config.pop(table) + def _del_key(self, config, table, key): with contextlib.suppress(KeyError): - config[self._table].pop(key) - if not config[self._table]: - config.pop(self._table) + config[table].pop(key) - with open(self._file, 'w') as stream: + if not config[table]: + self._del_table(config, table) + + def _write_config(self, config): + config = sonic_cfggen.FormatConverter.to_serialized(config) + with open(self._filepath, 'w') as stream: json.dump(config, stream, indent=4) @@ -64,37 +93,54 @@ class SonicDB: running DB and also for persistent and initial configs. """ - _running = None + _running_db_conn = None + + @classmethod + def get_connectors(cls): + """ Yields available DBs connectors. """ + + initial_db_conn = cls.get_initial_db_connector() + persistent_db_conn = cls.get_persistent_db_connector() + running_db_conn = cls.get_running_db_connector() + + yield initial_db_conn + if persistent_db_conn is not None: + yield persistent_db_conn + if running_db_conn is not None: + yield running_db_conn @classmethod - def running_table(cls, table): - """ Returns running DB table. """ + def get_running_db_connector(cls): + """ Returns running DB connector. """ # In chroot we can connect to a running # DB via TCP socket, we should ignore this case. if in_chroot(): return None - if cls._running is None: + if cls._running_db_conn is None: try: - cls._running = swsscommon.DBConnector(CONFIG_DB, 0) + cls._running_db_conn = swsscommon.ConfigDBConnector() + cls._running_db_conn.connect() except RuntimeError: # Failed to connect to DB. - return None + cls._running_db_conn = None - return swsscommon.Table(cls._running, table) + return cls._running_db_conn @classmethod - def persistent_table(cls, table): - """ Returns persistent DB table. """ + def get_persistent_db_connector(cls): + """ Returns persistent DB connector. """ if not os.path.exists(CONFIG_DB_JSON): return None - return FileDbTable(CONFIG_DB_JSON, table) + conn = PersistentConfigDbConnector(CONFIG_DB_JSON) + return conn @classmethod - def initial_table(cls, table): - """ Returns initial DB table. """ + def get_initial_db_connector(cls): + """ Returns initial DB connector. """ - return FileDbTable(INIT_CFG_JSON, table) + conn = PersistentConfigDbConnector(INIT_CFG_JSON) + return conn diff --git a/tests/sonic_package_manager/conftest.py b/tests/sonic_package_manager/conftest.py index 2c1a4be273..f63183b0c6 100644 --- a/tests/sonic_package_manager/conftest.py +++ b/tests/sonic_package_manager/conftest.py @@ -64,7 +64,7 @@ def mock_service_creator(): @pytest.fixture def mock_sonic_db(): - yield Mock() + yield MagicMock() @pytest.fixture diff --git a/tests/sonic_package_manager/test_service_creator.py b/tests/sonic_package_manager/test_service_creator.py index 07d1eec7ec..77b9c1ad61 100644 --- a/tests/sonic_package_manager/test_service_creator.py +++ b/tests/sonic_package_manager/test_service_creator.py @@ -115,13 +115,16 @@ def test_service_creator_with_debug_dump(sonic_fs, manifest, mock_feature_regist def test_service_creator_yang(sonic_fs, manifest, mock_feature_registry, - mock_sonic_db, mock_config_mgmt): - mock_table = Mock() + mock_sonic_db, mock_config_mgmt): test_yang = 'TEST YANG' - mock_table.get = Mock(return_value=(True, (('field_2', 'original_value_2'),))) - mock_sonic_db.initial_table = Mock(return_value=mock_table) - mock_sonic_db.persistent_table = Mock(return_value=mock_table) - mock_sonic_db.running_table = Mock(return_value=mock_table) + test_yang_module = 'sonic-test' + + mock_connector = Mock() + mock_sonic_db.get_connectors = Mock(return_value=[mock_connector]) + mock_connector.get_table = Mock(return_value={'key_a': {'field_1': 'value_1'}}) + mock_connector.get_config = Mock(return_value={ + 'TABLE_A': mock_connector.get_table('') + }) creator = ServiceCreator(mock_feature_registry, mock_sonic_db, mock_config_mgmt) @@ -129,13 +132,13 @@ def test_service_creator_yang(sonic_fs, manifest, mock_feature_registry, package = Package(entry, Metadata(manifest, yang_module_text=test_yang)) creator.create(package) - mock_config_mgmt.add_module.assert_called_with('TEST YANG') - mock_config_mgmt.get_module_name = Mock(return_value='sonic-test') + mock_config_mgmt.add_module.assert_called_with(test_yang) + mock_config_mgmt.get_module_name = Mock(return_value=test_yang_module) manifest['package']['init-cfg'] = { 'TABLE_A': { 'key_a': { - 'field_1': 'value_1', + 'field_1': 'new_value_1', 'field_2': 'value_2' }, }, @@ -146,72 +149,73 @@ def test_service_creator_yang(sonic_fs, manifest, mock_feature_registry, mock_config_mgmt.add_module.assert_called_with('TEST YANG') - mock_table.set.assert_called_with('key_a', [('field_1', 'value_1'), - ('field_2', 'original_value_2')]) + mock_connector.mod_config.assert_called_with( + { + 'TABLE_A': { + 'key_a': { + 'field_1': 'value_1', + 'field_2': 'value_2', + }, + }, + } + ) mock_config_mgmt.sy.confDbYangMap = { - 'TABLE_A': {'module': 'sonic-test'} + 'TABLE_A': {'module': test_yang_module} } - mock_table.getKeys = Mock(return_value=['key_a']) creator.remove(package) - mock_table._del.assert_called_with('key_a') - mock_config_mgmt.remove_module.assert_called_with('sonic-test') + mock_connector.set_entry.assert_called_with('TABLE_A', 'key_a', None) + mock_config_mgmt.remove_module.assert_called_with(test_yang_module) def test_feature_registration(mock_sonic_db, manifest): - mock_feature_table = Mock() - mock_feature_table.get = Mock(return_value=(False, ())) - mock_sonic_db.initial_table = Mock(return_value=mock_feature_table) - mock_sonic_db.persistent_table = Mock(return_value=mock_feature_table) - mock_sonic_db.running_table = Mock(return_value=mock_feature_table) + mock_connector = Mock() + mock_connector.get_entry = Mock(return_value={}) + mock_sonic_db.get_connectors = Mock(return_value=[mock_connector]) feature_registry = FeatureRegistry(mock_sonic_db) feature_registry.register(manifest) - mock_feature_table.set.assert_called_with('test', [ - ('state', 'disabled'), - ('auto_restart', 'enabled'), - ('high_mem_alert', 'disabled'), - ('set_owner', 'local'), - ('has_per_asic_scope', 'False'), - ('has_global_scope', 'True'), - ('has_timer', 'False'), - ]) + mock_connector.set_entry.assert_called_with('FEATURE', 'test', { + 'state': 'disabled', + 'auto_restart': 'enabled', + 'high_mem_alert': 'disabled', + 'set_owner': 'local', + 'has_per_asic_scope': 'False', + 'has_global_scope': 'True', + 'has_timer': 'False', + }) def test_feature_registration_with_timer(mock_sonic_db, manifest): manifest['service']['delayed'] = True - mock_feature_table = Mock() - mock_feature_table.get = Mock(return_value=(False, ())) - mock_sonic_db.initial_table = Mock(return_value=mock_feature_table) - mock_sonic_db.persistent_table = Mock(return_value=mock_feature_table) - mock_sonic_db.running_table = Mock(return_value=mock_feature_table) + mock_connector = Mock() + mock_connector.get_entry = Mock(return_value={}) + mock_sonic_db.get_connectors = Mock(return_value=[mock_connector]) feature_registry = FeatureRegistry(mock_sonic_db) feature_registry.register(manifest) - mock_feature_table.set.assert_called_with('test', [ - ('state', 'disabled'), - ('auto_restart', 'enabled'), - ('high_mem_alert', 'disabled'), - ('set_owner', 'local'), - ('has_per_asic_scope', 'False'), - ('has_global_scope', 'True'), - ('has_timer', 'True'), - ]) + mock_connector.set_entry.assert_called_with('FEATURE', 'test', { + 'state': 'disabled', + 'auto_restart': 'enabled', + 'high_mem_alert': 'disabled', + 'set_owner': 'local', + 'has_per_asic_scope': 'False', + 'has_global_scope': 'True', + 'has_timer': 'True', + }) def test_feature_registration_with_non_default_owner(mock_sonic_db, manifest): - mock_feature_table = Mock() - mock_feature_table.get = Mock(return_value=(False, ())) - mock_sonic_db.initial_table = Mock(return_value=mock_feature_table) - mock_sonic_db.persistent_table = Mock(return_value=mock_feature_table) - mock_sonic_db.running_table = Mock(return_value=mock_feature_table) + mock_connector = Mock() + mock_connector.get_entry = Mock(return_value={}) + mock_sonic_db.get_connectors = Mock(return_value=[mock_connector]) feature_registry = FeatureRegistry(mock_sonic_db) feature_registry.register(manifest, owner='kube') - mock_feature_table.set.assert_called_with('test', [ - ('state', 'disabled'), - ('auto_restart', 'enabled'), - ('high_mem_alert', 'disabled'), - ('set_owner', 'kube'), - ('has_per_asic_scope', 'False'), - ('has_global_scope', 'True'), - ('has_timer', 'False'), - ]) + mock_connector.set_entry.assert_called_with('FEATURE', 'test', { + 'state': 'disabled', + 'auto_restart': 'enabled', + 'high_mem_alert': 'disabled', + 'set_owner': 'kube', + 'has_per_asic_scope': 'False', + 'has_global_scope': 'True', + 'has_timer': 'False', + }) From 0a78e19ea3ea863905ed49e9d9c9c867ebec94af Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Thu, 13 May 2021 16:36:33 +0000 Subject: [PATCH 093/173] Removed arguments from constructor of class CliGenerator, added function comments Signed-off-by: Vadym Hlushko --- sonic_cli_gen/generator.py | 7 ++-- sonic_cli_gen/main.py | 4 +-- sonic_cli_gen/yang_parser.py | 70 ++++++++++++++++++++++++++---------- 3 files changed, 56 insertions(+), 25 deletions(-) diff --git a/sonic_cli_gen/generator.py b/sonic_cli_gen/generator.py index 0a02a104bc..6214bbe9c5 100644 --- a/sonic_cli_gen/generator.py +++ b/sonic_cli_gen/generator.py @@ -12,18 +12,17 @@ class CliGenerator: show, sonic-clear CLI plugins """ - def __init__(self, - yang_model): + def __init__(self): """ Initialize PackageManager. """ - self.yang_model_name = yang_model self.loader = jinja2.FileSystemLoader(['/usr/share/sonic/templates/sonic-cli-gen/']) self.env = jinja2.Environment(loader=self.loader) def generate_cli_plugin(self, cli_group, plugin_name): """ Generate CLI plugin. """ - parser = YangParser(self.yang_model_name) + parser = YangParser(plugin_name) yang_dict = parser.parse_yang_model() + #import pprint; pprint.pprint(yang_dict) plugin_path = get_cli_plugin_path(cli_group, plugin_name + '_yang.py') template = self.env.get_template(cli_group + '.py.j2') with open(plugin_path, 'w') as plugin_py: diff --git a/sonic_cli_gen/main.py b/sonic_cli_gen/main.py index 8cf748e394..0ee0b030f5 100644 --- a/sonic_cli_gen/main.py +++ b/sonic_cli_gen/main.py @@ -14,7 +14,7 @@ def cli(ctx): @click.pass_context def generate_config(ctx, yang_model_name): """ Generate CLI plugin (click) for 'config' CLI group. """ - gen = CliGenerator(yang_model_name) + gen = CliGenerator() gen.generate_cli_plugin(cli_group='config', plugin_name=yang_model_name) @cli.command() @@ -22,7 +22,7 @@ def generate_config(ctx, yang_model_name): @click.pass_context def generate_show(ctx, yang_model_name): """ Generate CLI plugin (click) for 'show' CLI group. """ - gen = CliGenerator(yang_model_name) + gen = CliGenerator() gen.generate_cli_plugin(cli_group='show', plugin_name=yang_model_name) if __name__ == '__main__': diff --git a/sonic_cli_gen/yang_parser.py b/sonic_cli_gen/yang_parser.py index a67a3d06d5..46bb147be6 100644 --- a/sonic_cli_gen/yang_parser.py +++ b/sonic_cli_gen/yang_parser.py @@ -14,7 +14,35 @@ class YangParser: y_top_level_container: Reference to top level 'container' entity from YANG model file y_table_containers: Reference to 'container' entities from YANG model file that represent Config DB tables - yang_2_dict: dictionary created from YANG model file that represent Config DB schema + yang_2_dict: dictionary created from YANG model file that represent Config DB schema. + In case if YANG model has a 'list' entity: + { + 'tables': [{ + 'name': 'value', + 'description': 'value', + 'dynamic-objects': [ + 'name': 'value', + 'description': 'value, + 'attrs': [ + { + 'name': 'value', + 'descruption': 'value', + 'is-leaf-list': False, + 'is-mandatory': False + } + ... + ], + 'keys': [ + { + 'name': 'ACL_TABLE_NAME', + 'description': 'value' + } + ... + ] + ], + }] + } + In case if YANG model does NOT have a 'list' entity, it has the same structure as above, but 'dynamic-objects' changed to 'static-objects' and have no 'keys' """ def __init__(self, yang_model_name): @@ -106,7 +134,6 @@ def on_table_container(y_module: OrderedDict, tbl_cont: OrderedDict) -> dict: # determine if 'table container' have a 'list' entity if tbl_cont.get('list') is None: - # TODO: comment aboit it y2d_elem['static-objects'] = list() # 'object' container goes after 'table' container @@ -120,12 +147,9 @@ def on_table_container(y_module: OrderedDict, tbl_cont: OrderedDict) -> dict: static_obj_elem = on_object_container(y_module, obj_cont, is_list=False) y2d_elem['static-objects'].append(static_obj_elem) else: - - # TODO: comment aboit it y2d_elem['dynamic-objects'] = list() - tbl_cont_lists = tbl_cont.get('list') - # 'container' can have more than 1 'list' + # 'container' can have more than 1 'list' entity if isinstance(tbl_cont_lists, list): for _list in tbl_cont_lists: dynamic_obj_elem = on_object_container(y_module, _list, is_list=True) @@ -134,21 +158,11 @@ def on_table_container(y_module: OrderedDict, tbl_cont: OrderedDict) -> dict: dynamic_obj_elem = on_object_container(y_module, tbl_cont_lists, is_list=True) y2d_elem['dynamic-objects'].append(dynamic_obj_elem) - # remove 'keys' elementsfrom 'attrs' list - remove_keys(y2d_elem['dynamic-objects']) + # move 'keys' elements from 'attrs' to 'keys' + change_dyn_obj_struct(y2d_elem['dynamic-objects']) return y2d_elem -# TODO: think about name -def remove_keys(dynamic_objects: OrderedDict): - for obj in dynamic_objects: - for key in obj.get('keys'): - for attr in obj.get('attrs'): - if key.get('name') == attr.get('name'): - key['description'] = attr.get('description') - obj['attrs'].remove(attr) - break - def on_object_container(y_module: OrderedDict, cont: OrderedDict, is_list: bool) -> dict: """ Parse a 'object container'. 'Object container' represent OBJECT inside Config DB schema: @@ -365,4 +379,22 @@ def get_list_keys(y_list: OrderedDict) -> list: key = { 'name': k } ret_list.append(key) - return ret_list \ No newline at end of file + return ret_list + +def change_dyn_obj_struct(dynamic_objects: OrderedDict): + """ Rearrange self.yang_2_dict['dynamic_objects'] structure. + If YANG model have a 'list' entity - inside the 'list' it has 'key' entity. + 'key' entity it is whitespace-separeted list of 'leafs', those 'leafs' was + parsed by 'on_leaf()' function and placed under 'attrs' in self.yang_2_dict['dynamic_objects'] + need to move 'leafs' from 'attrs' and put them to 'keys' section of elf.yang_2_dict['dynamic_objects'] + + Args: + dynamic_objects: reference to self.yang_2_dict['dynamic_objects'] + """ + for obj in dynamic_objects: + for key in obj.get('keys'): + for attr in obj.get('attrs'): + if key.get('name') == attr.get('name'): + key['description'] = attr.get('description') + obj['attrs'].remove(attr) + break \ No newline at end of file From f0fdbe2d27ac433f924d314828b9aa5930952ae8 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Fri, 14 May 2021 07:30:40 +0300 Subject: [PATCH 094/173] integrate with cli generation tool Signed-off-by: Stepan Blyschak --- sonic_package_manager/manager.py | 140 ++++++++++++++---- sonic_package_manager/manifest.py | 5 +- .../service_creator/creator.py | 5 +- tests/sonic_package_manager/conftest.py | 7 + .../test_service_creator.py | 41 +++-- 5 files changed, 145 insertions(+), 53 deletions(-) diff --git a/sonic_package_manager/manager.py b/sonic_package_manager/manager.py index f75c317e4f..c301b0ec22 100644 --- a/sonic_package_manager/manager.py +++ b/sonic_package_manager/manager.py @@ -2,6 +2,7 @@ import contextlib import functools +import hashlib import os import pkgutil import tempfile @@ -13,6 +14,8 @@ from config import config_mgmt from sonic_py_common import device_info +from sonic_cli_gen.generator import CliGenerator + from sonic_package_manager import utils from sonic_package_manager.constraint import ( VersionConstraint, @@ -53,7 +56,6 @@ RegistrySource, TarballSource ) -from sonic_package_manager.utils import DockerReference from sonic_package_manager.version import ( Version, VersionRange, @@ -62,6 +64,9 @@ ) +SONIC_CLI_COMMANDS = ('show', 'config', 'clear') + + @contextlib.contextmanager def failure_ignore(ignore: bool): """ Ignores failures based on parameter passed. """ @@ -103,7 +108,7 @@ def wrapped_function(*args, **kwargs): return wrapped_function -def rollback(func, *args, **kwargs): +def rollback(func, *args, **kwargs) -> Callable: """ Used in rollback callbacks to ignore failure but proceed with rollback. Error will be printed but not fail the whole procedure of rollback. """ @@ -132,7 +137,7 @@ def package_constraint_to_reference(constraint: PackageConstraint) -> PackageRef return PackageReference(package_name, version_to_tag(version_constraint)) -def parse_reference_expression(expression): +def parse_reference_expression(expression) -> PackageReference(name): try: return package_constraint_to_reference(PackageConstraint.parse(expression)) except ValueError: @@ -141,6 +146,54 @@ def parse_reference_expression(expression): return PackageReference.parse(expression) +def make_python_identifier(package: Package) -> str: + """ Generate unique python identifier from package name. + E.g: "sonic-package" and "sonic_package" are both valid package names, + while having single pythonized name "sonic_package". Hence, this function + calculates sha1 of package name and appends to the pythonized name. + + Args: + package: Package to generate python identifier for. + Returns: + Valid python identifier, unique for every package. + """ + + pythonized = utils.make_python_identifier(package.name) + return pythonized + hashlib.sha1(package.name.encode()).hexdigest() + + +def get_cli_plugin_directory(command: str) -> str: + """ Returns a plugins package directory for command group. + + Args: + command: SONiC command: "show"/"config"/"clear". + Returns: + Path to plugins package directory. + """ + + pkg_loader = pkgutil.get_loader(f'{command}.plugins') + if pkg_loader is None: + raise PackageManagerError(f'Failed to get plugins path for {command} CLI') + plugins_pkg_path = os.path.dirname(pkg_loader.path) + return plugins_pkg_path + + +def get_cli_plugin_path(package: Package, command: str, suffix: str = '') -> str: + """ Returns a path where to put CLI plugin code. + + Args: + package: Package to generate this path for. + command: SONiC command: "show"/"config"/"clear". + suffix: Optional suffix for python plugin name. + Returns: + Path generated for this package. + """ + + plugin_module_name = make_python_identifier(package) + '_' + suffix + plugin_module_file = plugin_module_name + '.py' + return os.path.join(get_cli_plugin_directory(command), plugin_module_file) + + def validate_package_base_os_constraints(package: Package, sonic_version_info: Dict[str, str]): """ Verify that all dependencies on base OS components are met. Args: @@ -257,6 +310,7 @@ def __init__(self, database: PackageDatabase, metadata_resolver: MetadataResolver, service_creator: ServiceCreator, + cli_generator: CliGenerator, device_information: Any, lock: filelock.FileLock): """ Initialize PackageManager. """ @@ -267,6 +321,7 @@ def __init__(self, self.database = database self.metadata_resolver = metadata_resolver self.service_creator = service_creator + self.cli_generator = cli_generator self.feature_registry = service_creator.feature_registry self.is_multi_npu = device_information.is_multi_npu() self.num_npus = device_information.get_num_npus() @@ -367,13 +422,18 @@ def install_from_source(self, # package name may not be in database. if not self.database.has_package(package.name): self.database.add_package(package.name, package.repository) + + service_create_opts = { + 'state': feature_state, + 'owner': default_owner, + } try: with contextlib.ExitStack() as exits: source.install(package) exits.callback(rollback(source.uninstall, package)) - self.service_creator.create(package, state=feature_state, owner=default_owner) + self.service_creator.create(package, **service_create_opts) exits.callback(rollback(self.service_creator.remove, package)) self.service_creator.generate_shutdown_sequence_files( @@ -522,6 +582,14 @@ def upgrade_from_source(self, validate_package_cli_can_be_skipped(new_package, skip_host_plugins) # After all checks are passed we proceed to actual upgrade + + service_create_opts = { + 'register_feature': False, + } + + service_remove_opts = { + 'register_feature': False, + } try: with contextlib.ExitStack() as exits: @@ -536,9 +604,9 @@ def upgrade_from_source(self, exits.callback(rollback(self._systemctl_action, old_package, 'start')) - self.service_creator.remove(old_package, deregister_feature=False) + self.service_creator.remove(old_package, **service_remove_opts) exits.callback(rollback(self.service_creator.create, old_package, - register_feature=False)) + **service_create_opts)) # Clean containers based on the old image containers = self.docker.ps(filters={'ancestor': old_package.image_id}, @@ -546,9 +614,9 @@ def upgrade_from_source(self, for container in containers: self.docker.rm(container.id, force=True) - self.service_creator.create(new_package, register_feature=False) + self.service_creator.create(new_package, **service_create_opts) exits.callback(rollback(self.service_creator.remove, new_package, - register_feature=False)) + **service_remove_opts)) self.service_creator.generate_shutdown_sequence_files( self._get_installed_packages_and(new_package) @@ -743,7 +811,7 @@ def get_package_source(self, ref = parse_reference_expression(package_expression) return self.get_package_source(package_ref=ref) elif repository_reference: - repo_ref = DockerReference.parse(repository_reference) + repo_ref = utils.DockerReference.parse(repository_reference) repository = repo_ref['name'] reference = repo_ref['tag'] or repo_ref['digest'] reference = reference or 'latest' @@ -906,41 +974,48 @@ def _systemctl_action(self, package: Package, action: str): for npu in range(self.num_npus): run_command(f'systemctl {action} {name}@{npu}') - @staticmethod - def _get_cli_plugin_name(package: Package): - return utils.make_python_identifier(package.name) + '.py' - - @classmethod - def _get_cli_plugin_path(cls, package: Package, command): - pkg_loader = pkgutil.get_loader(f'{command}.plugins') - if pkg_loader is None: - raise PackageManagerError(f'Failed to get plugins path for {command} CLI') - plugins_pkg_path = os.path.dirname(pkg_loader.path) - return os.path.join(plugins_pkg_path, cls._get_cli_plugin_name(package)) - def _install_cli_plugins(self, package: Package): - for command in ('show', 'config', 'clear'): + for command in SONIC_CLI_COMMANDS: self._install_cli_plugin(package, command) + self._install_autogen_cli(package, command) def _uninstall_cli_plugins(self, package: Package): - for command in ('show', 'config', 'clear'): + for command in SONIC_CLI_COMMANDS: self._uninstall_cli_plugin(package, command) + self._uninstall_autogen_cli(package, command) def _install_cli_plugin(self, package: Package, command: str): image_plugin_path = package.manifest['cli'][command] if not image_plugin_path: return - host_plugin_path = self._get_cli_plugin_path(package, command) + host_plugin_path = get_cli_plugin_path(package, command) self.docker.extract(package.entry.image_id, image_plugin_path, host_plugin_path) def _uninstall_cli_plugin(self, package: Package, command: str): image_plugin_path = package.manifest['cli'][command] if not image_plugin_path: return - host_plugin_path = self._get_cli_plugin_path(package, command) + host_plugin_path = get_cli_plugin_path(package, command) if os.path.exists(host_plugin_path): os.remove(host_plugin_path) + def _install_autogen_cli(self, package: Package, command: str): + if package.metadata.yang_module_text is None: + return + if not package.manifest['cli'][f'auto-generate-{command}']: + return + cfg_mgmt = self.service_creator.cfg_mgmt + module_name = cfg_mgmt.get_module_name(package.metadata.yang_module_text) + + plugin_path = get_cli_plugin_path(package, command, 'auto') + with open(plugin_path, 'w') as out: + self.cli_generator.generate_cli_plugin(module_name, command, out) + + def _uninstall_autogen_cli(self, package: Package, command: str): + plugin_path = get_cli_plugin_path(package, command, 'auto') + if os.path.exists(plugin_path): + os.remove(plugin_path) + @staticmethod def get_manager() -> 'PackageManager': """ Creates and returns PackageManager instance. @@ -949,13 +1024,20 @@ def get_manager() -> 'PackageManager': PackageManager """ - docker_api = DockerApi(docker.from_env()) + docker_api = DockerApi(docker.from_env(), ProgressManager()) registry_resolver = RegistryResolver() + metadata_resolver = MetadataResolver(docker_api, registry_resolver) cfg_mgmt = config_mgmt.ConfigMgmt() - return PackageManager(DockerApi(docker.from_env(), ProgressManager()), + cli_generator = CliGenerator() + sonic_db = SonicDB() + feautre_registry = FeatureRegistry(sonic_db) + service_creator = ServiceCreator(feature_registry, sonic_db, cfg_mgmt), + + return PackageManager(docker_api, registry_resolver, PackageDatabase.from_file(), - MetadataResolver(docker_api, registry_resolver), - ServiceCreator(FeatureRegistry(SonicDB), SonicDB, cfg_mgmt), + metadata_resolver, + service_creator, + cli_generator, device_info, filelock.FileLock(PACKAGE_MANAGER_LOCK_FILE, timeout=0)) diff --git a/sonic_package_manager/manifest.py b/sonic_package_manager/manifest.py index c126e2eef1..b234d26ee8 100644 --- a/sonic_package_manager/manifest.py +++ b/sonic_package_manager/manifest.py @@ -205,7 +205,10 @@ def unmarshal(self, value): ManifestField('mandatory', DefaultMarshaller(bool), False), ManifestField('show', DefaultMarshaller(str), ''), ManifestField('config', DefaultMarshaller(str), ''), - ManifestField('clear', DefaultMarshaller(str), '') + ManifestField('clear', DefaultMarshaller(str), ''), + ManifestField('auto-generate-show', DefaultMarshaller(bool), False), + ManifestField('auto-generate-config', DefaultMarshaller(bool), False), + ManifestField('auto-generate-clear', DefaultMarshaller(bool), False), ]) ]) diff --git a/sonic_package_manager/service_creator/creator.py b/sonic_package_manager/service_creator/creator.py index 4125bf8802..74b1faa596 100644 --- a/sonic_package_manager/service_creator/creator.py +++ b/sonic_package_manager/service_creator/creator.py @@ -4,6 +4,7 @@ import os import stat import subprocess +import pkgutil from collections import defaultdict from typing import Dict, Type @@ -138,7 +139,8 @@ def create(self, package: Package, register_feature: bool = True, state: str = 'enabled', - owner: str = 'local'): + owner: str = 'local', + skip_host_plugins: bool = False): """ Register package as SONiC service. Args: @@ -146,6 +148,7 @@ def create(self, register_feature: Wether to register this package in FEATURE table. state: Default feature state. owner: Default feature owner. + skip_host_plugins: Do not install host plugins from image if set to True. Returns: None diff --git a/tests/sonic_package_manager/conftest.py b/tests/sonic_package_manager/conftest.py index f63183b0c6..9f1fc0c966 100644 --- a/tests/sonic_package_manager/conftest.py +++ b/tests/sonic_package_manager/conftest.py @@ -72,6 +72,11 @@ def mock_config_mgmt(): yield MagicMock() +@pytest.fixture +def mock_cli_gen(): + yield MagicMock() + + @pytest.fixture def fake_metadata_resolver(): class FakeMetadataResolver: @@ -421,6 +426,7 @@ def patch_pkgutil(): def package_manager(mock_docker_api, mock_registry_resolver, mock_service_creator, + mock_cli_gen, mock_config_mgmt, fake_metadata_resolver, fake_db, @@ -428,6 +434,7 @@ def package_manager(mock_docker_api, yield PackageManager(mock_docker_api, mock_registry_resolver, fake_db, fake_metadata_resolver, mock_service_creator, + mock_cli_gen, fake_device_info, MagicMock()) diff --git a/tests/sonic_package_manager/test_service_creator.py b/tests/sonic_package_manager/test_service_creator.py index 77b9c1ad61..c24dcea157 100644 --- a/tests/sonic_package_manager/test_service_creator.py +++ b/tests/sonic_package_manager/test_service_creator.py @@ -59,14 +59,17 @@ def manifest(): }) -def test_service_creator(sonic_fs, manifest, mock_feature_registry, - mock_sonic_db, mock_config_mgmt, package_manager): - creator = ServiceCreator(mock_feature_registry, mock_sonic_db, mock_config_mgmt) +@pytest.fixture() +def service_creator(mock_feature_registry, mock_sonic_db, mock_config_mgmt): + yield ServiceCreator(mock_feature_registry, mock_sonic_db, mock_config_mgmt) + + +def test_service_creator(sonic_fs, manifest, service_creator, package_manager): entry = PackageEntry('test', 'azure/sonic-test') package = Package(entry, Metadata(manifest)) installed_packages = package_manager._get_installed_packages_and(package) - creator.create(package) - creator.generate_shutdown_sequence_files(installed_packages) + service_creator.create(package) + service_creator.generate_shutdown_sequence_files(installed_packages) assert sonic_fs.exists(os.path.join(ETC_SONIC_PATH, 'swss_dependent')) assert sonic_fs.exists(os.path.join(DOCKER_CTL_SCRIPT_LOCATION, 'test.sh')) @@ -82,40 +85,36 @@ def read_file(name): assert read_file('test_reconcile') == 'test-process test-process-3' -def test_service_creator_with_timer_unit(sonic_fs, manifest, mock_feature_registry, - mock_sonic_db, mock_config_mgmt): - creator = ServiceCreator(mock_feature_registry, mock_sonic_db, mock_config_mgmt) +def test_service_creator_with_timer_unit(sonic_fs, manifest, service_creator): entry = PackageEntry('test', 'azure/sonic-test') package = Package(entry, Metadata(manifest)) - creator.create(package) + service_creator.create(package) assert not sonic_fs.exists(os.path.join(SYSTEMD_LOCATION, 'test.timer')) manifest['service']['delayed'] = True package = Package(entry, Metadata(manifest)) - creator.create(package) + service_creator.create(package) assert sonic_fs.exists(os.path.join(SYSTEMD_LOCATION, 'test.timer')) -def test_service_creator_with_debug_dump(sonic_fs, manifest, mock_feature_registry, - mock_sonic_db, mock_config_mgmt): - creator = ServiceCreator(mock_feature_registry, mock_sonic_db, mock_config_mgmt) +def test_service_creator_with_debug_dump(sonic_fs, manifest, service_creator): entry = PackageEntry('test', 'azure/sonic-test') package = Package(entry, Metadata(manifest)) - creator.create(package) + service_creator.create(package) assert not sonic_fs.exists(os.path.join(DEBUG_DUMP_SCRIPT_LOCATION, 'test')) manifest['package']['debug-dump'] = '/some/command' package = Package(entry, Metadata(manifest)) - creator.create(package) + service_creator.create(package) assert sonic_fs.exists(os.path.join(DEBUG_DUMP_SCRIPT_LOCATION, 'test')) -def test_service_creator_yang(sonic_fs, manifest, mock_feature_registry, - mock_sonic_db, mock_config_mgmt): +def test_service_creator_yang(sonic_fs, manifest, mock_sonic_db, + mock_config_mgmt, service_creator): test_yang = 'TEST YANG' test_yang_module = 'sonic-test' @@ -126,11 +125,9 @@ def test_service_creator_yang(sonic_fs, manifest, mock_feature_registry, 'TABLE_A': mock_connector.get_table('') }) - creator = ServiceCreator(mock_feature_registry, mock_sonic_db, mock_config_mgmt) - entry = PackageEntry('test', 'azure/sonic-test') package = Package(entry, Metadata(manifest, yang_module_text=test_yang)) - creator.create(package) + service_creator.create(package) mock_config_mgmt.add_module.assert_called_with(test_yang) mock_config_mgmt.get_module_name = Mock(return_value=test_yang_module) @@ -145,7 +142,7 @@ def test_service_creator_yang(sonic_fs, manifest, mock_feature_registry, } package = Package(entry, Metadata(manifest, yang_module_text=test_yang)) - creator.create(package) + service_creator.create(package) mock_config_mgmt.add_module.assert_called_with('TEST YANG') @@ -164,7 +161,7 @@ def test_service_creator_yang(sonic_fs, manifest, mock_feature_registry, 'TABLE_A': {'module': test_yang_module} } - creator.remove(package) + service_creator.remove(package) mock_connector.set_entry.assert_called_with('TABLE_A', 'key_a', None) mock_config_mgmt.remove_module.assert_called_with(test_yang_module) From 5784d6cfe722f47685114c5e9384f4a3dc5ea8bc Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Fri, 14 May 2021 07:59:56 +0300 Subject: [PATCH 095/173] fix manager Signed-off-by: Stepan Blyschak --- sonic_package_manager/manager.py | 2 +- sonic_package_manager/service_creator/creator.py | 4 +--- tests/sonic_package_manager/conftest.py | 9 +++++---- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/sonic_package_manager/manager.py b/sonic_package_manager/manager.py index c301b0ec22..f6777b3592 100644 --- a/sonic_package_manager/manager.py +++ b/sonic_package_manager/manager.py @@ -137,7 +137,7 @@ def package_constraint_to_reference(constraint: PackageConstraint) -> PackageRef return PackageReference(package_name, version_to_tag(version_constraint)) -def parse_reference_expression(expression) -> PackageReference(name): +def parse_reference_expression(expression) -> PackageReference: try: return package_constraint_to_reference(PackageConstraint.parse(expression)) except ValueError: diff --git a/sonic_package_manager/service_creator/creator.py b/sonic_package_manager/service_creator/creator.py index 74b1faa596..162d3afe0a 100644 --- a/sonic_package_manager/service_creator/creator.py +++ b/sonic_package_manager/service_creator/creator.py @@ -139,8 +139,7 @@ def create(self, package: Package, register_feature: bool = True, state: str = 'enabled', - owner: str = 'local', - skip_host_plugins: bool = False): + owner: str = 'local'): """ Register package as SONiC service. Args: @@ -148,7 +147,6 @@ def create(self, register_feature: Wether to register this package in FEATURE table. state: Default feature state. owner: Default feature owner. - skip_host_plugins: Do not install host plugins from image if set to True. Returns: None diff --git a/tests/sonic_package_manager/conftest.py b/tests/sonic_package_manager/conftest.py index 9f1fc0c966..e5bdebb4c4 100644 --- a/tests/sonic_package_manager/conftest.py +++ b/tests/sonic_package_manager/conftest.py @@ -119,7 +119,7 @@ def __init__(self): 'before': ['swss'], } ) - self.add('Azure/docker-test', '1.6.0', 'test-package', '1.6.0') + self.add('Azure/docker-test', '1.6.0', 'test-package', '1.6.0', yang='TEST') self.add('Azure/docker-test-2', '1.5.0', 'test-package-2', '1.5.0') self.add('Azure/docker-test-2', '2.0.0', 'test-package-2', '2.0.0') self.add('Azure/docker-test-3', 'latest', 'test-package-3', '1.6.0') @@ -268,7 +268,7 @@ def fake_db(fake_metadata_resolver): description='SONiC Package Manager Test Package', default_reference='1.6.0', installed=False, - built_in=False + built_in=False, ) add_package( content, @@ -418,8 +418,9 @@ def sonic_fs(fs): @pytest.fixture(autouse=True) def patch_pkgutil(): - with mock.patch('pkgutil.get_loader'): - yield + yield + # with mock.patch('pkgutil.get_loader') as loader: + # yield loader @pytest.fixture From 65b125c495af878a9092ea2f37867f8a6e045e08 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Fri, 14 May 2021 08:00:13 +0300 Subject: [PATCH 096/173] [sonic-cli-gen] Signed-off-by: Stepan Blyschak --- .../templates/sonic-cli-gen/config.py.j2 | 2 +- sonic_cli_gen/generator.py | 19 +++++++++++-------- sonic_cli_gen/main.py | 8 ++++---- 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 b/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 index ac19835cd1..89580539f5 100644 --- a/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 +++ b/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 @@ -396,7 +396,7 @@ def {{ group }}_delete(db, {{ pythonize(object["keys"]) }}): {# Generate another nesting group in case table holds two types of objects #} {% if table["dynamic-objects"]|length > 1 %} {% set group = table.name + "_" + object.name %} -@{{ table.name }}.group(name="{{ cli_name(object.name) }}", +@{{ table.name }}.group(name="{{ cli_name(object.name|replace(table.name + "_", "", 1)) }}", cls=clicommon.AliasedGroup) def {{ group }}(): """ {{ object.description }} """ diff --git a/sonic_cli_gen/generator.py b/sonic_cli_gen/generator.py index 0a02a104bc..199149ff73 100644 --- a/sonic_cli_gen/generator.py +++ b/sonic_cli_gen/generator.py @@ -12,22 +12,25 @@ class CliGenerator: show, sonic-clear CLI plugins """ - def __init__(self, - yang_model): + def __init__(self): """ Initialize PackageManager. """ - self.yang_model_name = yang_model self.loader = jinja2.FileSystemLoader(['/usr/share/sonic/templates/sonic-cli-gen/']) self.env = jinja2.Environment(loader=self.loader) - def generate_cli_plugin(self, cli_group, plugin_name): + def generate_cli_plugin(self, yang_model_name, cli_group, output_stream=None): """ Generate CLI plugin. """ - parser = YangParser(self.yang_model_name) + parser = YangParser(yang_model_name) yang_dict = parser.parse_yang_model() - plugin_path = get_cli_plugin_path(cli_group, plugin_name + '_yang.py') template = self.env.get_template(cli_group + '.py.j2') - with open(plugin_path, 'w') as plugin_py: - plugin_py.write(template.render(yang_dict)) + + if output_stream is None: + plugin_name = yang_module_name.replace('-', '_') + plugin_path = get_cli_plugin_path(cli_group, plugin_name + '_auto.py') + with open(plugin_path, 'w') as output_stream: + output_stream.write(template.render(yang_dict)) + else: + output_stream.write(template.render(yang_dict)) def get_cli_plugin_path(command, plugin_name): pkg_loader = pkgutil.get_loader(f'{command}.plugins') diff --git a/sonic_cli_gen/main.py b/sonic_cli_gen/main.py index 8cf748e394..6e1b1874e8 100644 --- a/sonic_cli_gen/main.py +++ b/sonic_cli_gen/main.py @@ -14,16 +14,16 @@ def cli(ctx): @click.pass_context def generate_config(ctx, yang_model_name): """ Generate CLI plugin (click) for 'config' CLI group. """ - gen = CliGenerator(yang_model_name) - gen.generate_cli_plugin(cli_group='config', plugin_name=yang_model_name) + gen = CliGenerator() + gen.generate_cli_plugin(yang_model_name, cli_group='config', plugin_name=yang_model_name) @cli.command() @click.argument('yang_model_name') @click.pass_context def generate_show(ctx, yang_model_name): """ Generate CLI plugin (click) for 'show' CLI group. """ - gen = CliGenerator(yang_model_name) - gen.generate_cli_plugin(cli_group='show', plugin_name=yang_model_name) + gen = CliGenerator() + gen.generate_cli_plugin(yang_model_name, cli_group='show', plugin_name=yang_model_name) if __name__ == '__main__': cli() \ No newline at end of file From 134b21e8f529b550834d54ea502341c644a1cf23 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Fri, 14 May 2021 09:52:39 +0300 Subject: [PATCH 097/173] [sonic-cli-gen] Signed-off-by: Stepan Blyschak --- sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 b/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 index 58e7dfd2e0..ea82b9761f 100644 --- a/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 +++ b/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 @@ -77,7 +77,8 @@ def {{ name }}(db): body = [] table = db.cfgdb.get_table("{{ table.name }}") - for key, entry in natsort.natsorted(table).items(): + for key in natsort.natsorted(table): + entry = table[key] if not isinstance(key, tuple): key = (key,) From 049823f1e8757181f46e6444cc261bea522ecea5 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Fri, 14 May 2021 09:52:59 +0300 Subject: [PATCH 098/173] manager fixes Signed-off-by: Stepan Blyschak --- sonic_package_manager/manager.py | 4 ++-- sonic_package_manager/service_creator/creator.py | 2 +- sonic_package_manager/service_creator/feature.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/sonic_package_manager/manager.py b/sonic_package_manager/manager.py index f6777b3592..24d505eb0c 100644 --- a/sonic_package_manager/manager.py +++ b/sonic_package_manager/manager.py @@ -1030,8 +1030,8 @@ def get_manager() -> 'PackageManager': cfg_mgmt = config_mgmt.ConfigMgmt() cli_generator = CliGenerator() sonic_db = SonicDB() - feautre_registry = FeatureRegistry(sonic_db) - service_creator = ServiceCreator(feature_registry, sonic_db, cfg_mgmt), + feature_registry = FeatureRegistry(sonic_db) + service_creator = ServiceCreator(feature_registry, sonic_db, cfg_mgmt) return PackageManager(docker_api, registry_resolver, diff --git a/sonic_package_manager/service_creator/creator.py b/sonic_package_manager/service_creator/creator.py index 162d3afe0a..a23769ef58 100644 --- a/sonic_package_manager/service_creator/creator.py +++ b/sonic_package_manager/service_creator/creator.py @@ -504,7 +504,7 @@ def remove_config(self, package): module_name = self.cfg_mgmt.get_module_name(package.metadata.yang_module_text) for tablename, module in self.cfg_mgmt.sy.confDbYangMap.items(): - if module['module'] != module_name: + if module.get('module') != module_name: continue for conn in self.sonic_db.get_connectors(): diff --git a/sonic_package_manager/service_creator/feature.py b/sonic_package_manager/service_creator/feature.py index 7127847073..dbc93a8284 100644 --- a/sonic_package_manager/service_creator/feature.py +++ b/sonic_package_manager/service_creator/feature.py @@ -83,7 +83,7 @@ def is_feature_enabled(self, name: str) -> bool: if conn is None: return False - cfg = conn.get_entry(name) + cfg = conn.get_entry(FEATURE, name) return is_enabled(cfg) def get_multi_instance_features(self): @@ -91,7 +91,7 @@ def get_multi_instance_features(self): conn = self._sonic_db.get_initial_db_connector() features = conn.get_table(FEATURE) - return [feature for feature, cfg in features if is_multi_instance(cfg)] + return [feature for feature, cfg in features.items() if is_multi_instance(cfg)] @staticmethod def get_default_feature_entries(state=None, owner=None) -> Dict[str, str]: From 30d2cb9c774db6df42121a0f486362f07602c5d8 Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Fri, 14 May 2021 15:17:36 +0000 Subject: [PATCH 099/173] Added sceleton for UT, added additional check for yang_parser.py, changed constructor for YangParser Signed-off-by: Vadym Hlushko --- sonic_cli_gen/generator.py | 7 +- sonic_cli_gen/yang_parser.py | 12 +- tests/cli_autogen_input/config_db.json | 544 ++++++++++++++++++ .../sonic-one-table-container.yang | 19 + tests/cli_autogen_input/sonic-vlan.yang | 190 ++++++ tests/cli_autogen_yang_parser_test.py | 56 ++ 6 files changed, 824 insertions(+), 4 deletions(-) create mode 100644 tests/cli_autogen_input/config_db.json create mode 100644 tests/cli_autogen_input/sonic-one-table-container.yang create mode 100644 tests/cli_autogen_input/sonic-vlan.yang create mode 100644 tests/cli_autogen_yang_parser_test.py diff --git a/sonic_cli_gen/generator.py b/sonic_cli_gen/generator.py index 6214bbe9c5..f3fcf0e62b 100644 --- a/sonic_cli_gen/generator.py +++ b/sonic_cli_gen/generator.py @@ -20,9 +20,12 @@ def __init__(self): def generate_cli_plugin(self, cli_group, plugin_name): """ Generate CLI plugin. """ - parser = YangParser(plugin_name) + + parser = YangParser(yang_model_name=plugin_name, + config_db_path='configDB', + allow_tbl_without_yang=True, + debug=False) yang_dict = parser.parse_yang_model() - #import pprint; pprint.pprint(yang_dict) plugin_path = get_cli_plugin_path(cli_group, plugin_name + '_yang.py') template = self.env.get_template(cli_group + '.py.j2') with open(plugin_path, 'w') as plugin_py: diff --git a/sonic_cli_gen/yang_parser.py b/sonic_cli_gen/yang_parser.py index 46bb147be6..b36d862ee8 100644 --- a/sonic_cli_gen/yang_parser.py +++ b/sonic_cli_gen/yang_parser.py @@ -45,7 +45,10 @@ class YangParser: In case if YANG model does NOT have a 'list' entity, it has the same structure as above, but 'dynamic-objects' changed to 'static-objects' and have no 'keys' """ def __init__(self, - yang_model_name): + yang_model_name, + config_db_path, + allow_tbl_without_yang, + debug): self.yang_model_name = yang_model_name self.conf_mgmt = None self.idx_yJson = None @@ -55,7 +58,9 @@ def __init__(self, self.yang_2_dict = dict() try: - self.conf_mgmt = ConfigMgmt() + self.conf_mgmt = ConfigMgmt(source=config_db_path, + debug=debug, + allowTablesWithoutYang=allow_tbl_without_yang) except Exception as e: raise Exception("Failed to load the {} class".format(str(e))) @@ -181,6 +186,9 @@ def on_object_container(y_module: OrderedDict, cont: OrderedDict, is_list: bool) dictionary - element for y2d_elem['static-objects'] OR y2d_elem['dynamic-objects'] """ + if cont is None: + return {} + obj_elem = { 'name': cont.get('@name'), 'description': get_description(cont), diff --git a/tests/cli_autogen_input/config_db.json b/tests/cli_autogen_input/config_db.json new file mode 100644 index 0000000000..5473d6158a --- /dev/null +++ b/tests/cli_autogen_input/config_db.json @@ -0,0 +1,544 @@ +{ + "COPP_GROUP": { + "default": { + "cbs": "600", + "cir": "600", + "meter_type": "packets", + "mode": "sr_tcm", + "queue": "0", + "red_action": "drop" + }, + "queue1_group1": { + "cbs": "6000", + "cir": "6000", + "meter_type": "packets", + "mode": "sr_tcm", + "queue": "1", + "red_action": "drop", + "trap_action": "trap", + "trap_priority": "1" + }, + "queue1_group2": { + "cbs": "600", + "cir": "600", + "meter_type": "packets", + "mode": "sr_tcm", + "queue": "1", + "red_action": "drop", + "trap_action": "trap", + "trap_priority": "1" + }, + "queue2_group1": { + "cbs": "1000", + "cir": "1000", + "genetlink_mcgrp_name": "packets", + "genetlink_name": "psample", + "meter_type": "packets", + "mode": "sr_tcm", + "queue": "2", + "red_action": "drop", + "trap_action": "trap", + "trap_priority": "1" + }, + "queue4_group1": { + "cbs": "600", + "cir": "600", + "color": "blind", + "meter_type": "packets", + "mode": "sr_tcm", + "queue": "4", + "red_action": "drop", + "trap_action": "trap", + "trap_priority": "4" + }, + "queue4_group2": { + "cbs": "600", + "cir": "600", + "meter_type": "packets", + "mode": "sr_tcm", + "queue": "4", + "red_action": "drop", + "trap_action": "copy", + "trap_priority": "4" + }, + "queue4_group3": { + "cbs": "600", + "cir": "600", + "color": "blind", + "meter_type": "packets", + "mode": "sr_tcm", + "queue": "4", + "red_action": "drop", + "trap_action": "trap", + "trap_priority": "4" + } + }, + "COPP_TRAP": { + "arp": { + "trap_group": "queue4_group2", + "trap_ids": "arp_req,arp_resp,neigh_discovery" + }, + "bgp": { + "trap_group": "queue4_group1", + "trap_ids": "bgp,bgpv6" + }, + "dhcp": { + "trap_group": "queue4_group3", + "trap_ids": "dhcp,dhcpv6" + }, + "ip2me": { + "trap_group": "queue1_group1", + "trap_ids": "ip2me" + }, + "lacp": { + "trap_group": "queue4_group1", + "trap_ids": "lacp" + }, + "lldp": { + "trap_group": "queue4_group3", + "trap_ids": "lldp" + }, + "nat": { + "trap_group": "queue1_group2", + "trap_ids": "src_nat_miss,dest_nat_miss" + }, + "sflow": { + "trap_group": "queue2_group1", + "trap_ids": "sample_packet" + }, + "ssh": { + "trap_group": "queue4_group2", + "trap_ids": "ssh" + }, + "udld": { + "trap_group": "queue4_group3", + "trap_ids": "udld" + } + }, + "CRM": { + "Config": { + "acl_counter_high_threshold": "85", + "acl_counter_low_threshold": "70", + "acl_counter_threshold_type": "percentage", + "acl_entry_high_threshold": "85", + "acl_entry_low_threshold": "70", + "acl_entry_threshold_type": "percentage", + "acl_group_high_threshold": "85", + "acl_group_low_threshold": "70", + "acl_group_threshold_type": "percentage", + "acl_table_high_threshold": "85", + "acl_table_low_threshold": "70", + "acl_table_threshold_type": "percentage", + "dnat_entry_high_threshold": "85", + "dnat_entry_low_threshold": "70", + "dnat_entry_threshold_type": "percentage", + "fdb_entry_high_threshold": "85", + "fdb_entry_low_threshold": "70", + "fdb_entry_threshold_type": "percentage", + "ipmc_entry_high_threshold": "85", + "ipmc_entry_low_threshold": "70", + "ipmc_entry_threshold_type": "percentage", + "ipv4_neighbor_high_threshold": "85", + "ipv4_neighbor_low_threshold": "70", + "ipv4_neighbor_threshold_type": "percentage", + "ipv4_nexthop_high_threshold": "85", + "ipv4_nexthop_low_threshold": "70", + "ipv4_nexthop_threshold_type": "percentage", + "ipv4_route_high_threshold": "85", + "ipv4_route_low_threshold": "70", + "ipv4_route_threshold_type": "percentage", + "ipv6_neighbor_high_threshold": "85", + "ipv6_neighbor_low_threshold": "70", + "ipv6_neighbor_threshold_type": "percentage", + "ipv6_nexthop_high_threshold": "85", + "ipv6_nexthop_low_threshold": "70", + "ipv6_nexthop_threshold_type": "percentage", + "ipv6_route_high_threshold": "85", + "ipv6_route_low_threshold": "70", + "ipv6_route_threshold_type": "percentage", + "nexthop_group_high_threshold": "85", + "nexthop_group_low_threshold": "70", + "nexthop_group_member_high_threshold": "85", + "nexthop_group_member_low_threshold": "70", + "nexthop_group_member_threshold_type": "percentage", + "nexthop_group_threshold_type": "percentage", + "polling_interval": "300", + "snat_entry_high_threshold": "85", + "snat_entry_low_threshold": "70", + "snat_entry_threshold_type": "percentage" + } + }, + "DEVICE_METADATA": { + "localhost": { + "buffer_model": "traditional", + "default_bgp_status": "up", + "default_pfcwd_status": "disable", + "hostname": "r-bulldog-02", + "hwsku": "ACS-MSN2100", + "mac": "98:03:9b:f8:e7:c0", + "platform": "x86_64-mlnx_msn2100-r0", + "type": "ToRRouter" + } + }, + "FEATURE": { + "bgp": { + "auto_restart": "enabled", + "has_global_scope": "False", + "has_per_asic_scope": "True", + "has_timer": "False", + "high_mem_alert": "disabled", + "state": "enabled" + }, + "database": { + "auto_restart": "disabled", + "has_global_scope": "True", + "has_per_asic_scope": "True", + "has_timer": "False", + "high_mem_alert": "disabled", + "state": "enabled" + }, + "dhcp_relay": { + "auto_restart": "enabled", + "has_global_scope": "True", + "has_per_asic_scope": "False", + "has_timer": "False", + "high_mem_alert": "disabled", + "state": "enabled" + }, + "lldp": { + "auto_restart": "enabled", + "has_global_scope": "True", + "has_per_asic_scope": "True", + "has_timer": "False", + "high_mem_alert": "disabled", + "state": "enabled", + "status": "enabled" + }, + "mgmt-framework": { + "auto_restart": "enabled", + "has_global_scope": "True", + "has_per_asic_scope": "False", + "has_timer": "True", + "high_mem_alert": "disabled", + "state": "enabled" + }, + "nat": { + "auto_restart": "enabled", + "has_global_scope": "True", + "has_per_asic_scope": "False", + "has_timer": "False", + "high_mem_alert": "disabled", + "state": "disabled" + }, + "pmon": { + "auto_restart": "enabled", + "has_global_scope": "True", + "has_per_asic_scope": "False", + "has_timer": "False", + "high_mem_alert": "disabled", + "state": "enabled" + }, + "radv": { + "auto_restart": "enabled", + "has_global_scope": "True", + "has_per_asic_scope": "False", + "has_timer": "False", + "high_mem_alert": "disabled", + "state": "enabled" + }, + "sflow": { + "auto_restart": "enabled", + "has_global_scope": "True", + "has_per_asic_scope": "False", + "has_timer": "False", + "high_mem_alert": "disabled", + "state": "disabled" + }, + "snmp": { + "auto_restart": "enabled", + "has_global_scope": "True", + "has_per_asic_scope": "False", + "has_timer": "True", + "high_mem_alert": "disabled", + "state": "enabled" + }, + "swss": { + "auto_restart": "enabled", + "has_global_scope": "False", + "has_per_asic_scope": "True", + "has_timer": "False", + "high_mem_alert": "disabled", + "state": "enabled" + }, + "syncd": { + "auto_restart": "enabled", + "has_global_scope": "False", + "has_per_asic_scope": "True", + "has_timer": "False", + "high_mem_alert": "disabled", + "state": "enabled" + }, + "teamd": { + "auto_restart": "enabled", + "has_global_scope": "False", + "has_per_asic_scope": "True", + "has_timer": "False", + "high_mem_alert": "disabled", + "state": "enabled" + }, + "telemetry": { + "auto_restart": "enabled", + "has_global_scope": "True", + "has_per_asic_scope": "False", + "has_timer": "True", + "high_mem_alert": "disabled", + "state": "enabled" + }, + "what-just-happened": { + "auto_restart": "disabled", + "has_timer": "True", + "high_mem_alert": "disabled", + "state": "enabled" + } + }, + "FLEX_COUNTER_TABLE": { + "BUFFER_POOL_WATERMARK": { + "FLEX_COUNTER_STATUS": "enable" + }, + "PFCWD": { + "FLEX_COUNTER_STATUS": "enable" + }, + "PG_WATERMARK": { + "FLEX_COUNTER_STATUS": "enable" + }, + "PORT": { + "FLEX_COUNTER_STATUS": "enable" + }, + "PORT_BUFFER_DROP": { + "FLEX_COUNTER_STATUS": "enable" + }, + "QUEUE": { + "FLEX_COUNTER_STATUS": "enable" + }, + "QUEUE_WATERMARK": { + "FLEX_COUNTER_STATUS": "enable" + }, + "RIF": { + "FLEX_COUNTER_STATUS": "enable" + } + }, + "KDUMP": { + "config": { + "enabled": "false", + "num_dumps": "3", + "memory": "0M-2G:256M,2G-4G:320M,4G-8G:384M,8G-:448M" + } + }, + "MGMT_INTERFACE": { + "eth0|10.210.25.44/22": { + "gwaddr": "10.210.24.1" + } + }, + "PORT": { + "Ethernet0": { + "admin_status": "up", + "alias": "etp1", + "index": "1", + "lanes": "0,1,2,3", + "speed": "100000" + }, + "Ethernet12": { + "admin_status": "up", + "alias": "etp4a", + "index": "4", + "lanes": "12,13", + "speed": "50000" + }, + "Ethernet14": { + "admin_status": "up", + "alias": "etp4b", + "index": "4", + "lanes": "14,15", + "speed": "50000" + }, + "Ethernet16": { + "admin_status": "up", + "alias": "etp5a", + "index": "5", + "lanes": "16,17", + "speed": "50000" + }, + "Ethernet18": { + "admin_status": "up", + "alias": "etp5b", + "index": "5", + "lanes": "18,19", + "speed": "50000" + }, + "Ethernet20": { + "admin_status": "up", + "alias": "etp6a", + "index": "6", + "lanes": "20", + "speed": "25000" + }, + "Ethernet21": { + "admin_status": "up", + "alias": "etp6b", + "index": "6", + "lanes": "21", + "speed": "25000" + }, + "Ethernet22": { + "admin_status": "up", + "alias": "etp6c", + "index": "6", + "lanes": "22", + "speed": "25000" + }, + "Ethernet23": { + "admin_status": "up", + "alias": "etp6d", + "index": "6", + "lanes": "23", + "speed": "25000" + }, + "Ethernet24": { + "admin_status": "up", + "alias": "etp7a", + "index": "7", + "lanes": "24", + "speed": "25000" + }, + "Ethernet25": { + "admin_status": "up", + "alias": "etp7b", + "index": "7", + "lanes": "25", + "speed": "25000" + }, + "Ethernet26": { + "admin_status": "up", + "alias": "etp7c", + "index": "7", + "lanes": "26", + "speed": "25000" + }, + "Ethernet27": { + "admin_status": "up", + "alias": "etp7d", + "index": "7", + "lanes": "27", + "speed": "25000" + }, + "Ethernet28": { + "admin_status": "up", + "alias": "etp8", + "index": "8", + "lanes": "28,29,30,31", + "speed": "100000" + }, + "Ethernet32": { + "admin_status": "up", + "alias": "etp9", + "index": "9", + "lanes": "32,33,34,35", + "speed": "100000" + }, + "Ethernet36": { + "admin_status": "up", + "alias": "etp10", + "index": "10", + "lanes": "36,37,38,39", + "speed": "100000" + }, + "Ethernet4": { + "admin_status": "up", + "alias": "etp2", + "index": "2", + "lanes": "4,5,6,7", + "speed": "100000" + }, + "Ethernet40": { + "admin_status": "up", + "alias": "etp11", + "index": "11", + "lanes": "40,41,42,43", + "speed": "100000" + }, + "Ethernet44": { + "admin_status": "up", + "alias": "etp12", + "index": "12", + "lanes": "44,45,46,47", + "speed": "100000" + }, + "Ethernet48": { + "admin_status": "up", + "alias": "etp13", + "index": "13", + "lanes": "48,49,50,51", + "speed": "100000" + }, + "Ethernet52": { + "admin_status": "up", + "alias": "etp14", + "index": "14", + "lanes": "52,53,54,55", + "speed": "100000" + }, + "Ethernet56": { + "admin_status": "up", + "alias": "etp15", + "index": "15", + "lanes": "56,57,58,59", + "speed": "100000" + }, + "Ethernet60": { + "admin_status": "up", + "alias": "etp16", + "index": "16", + "lanes": "60,61,62,63", + "speed": "100000" + }, + "Ethernet8": { + "admin_status": "up", + "alias": "etp3", + "index": "3", + "lanes": "8,9,10,11", + "speed": "100000" + } + }, + "SNMP": { + "LOCATION": { + "Location": "public" + } + }, + "SNMP_COMMUNITY": { + "public": { + "TYPE": "RO" + } + }, + "VERSIONS": { + "DATABASE": { + "VERSION": "version_2_0_0" + } + }, + "WJH": { + "global": { + "mode": "debug", + "nice_level": "1", + "pci_bandwidth": "50" + } + }, + "WJH_CHANNEL": { + "forwarding": { + "drop_category_list": "L2,L3,Tunnel", + "type": "raw_and_aggregated" + }, + "layer-1": { + "drop_category_list": "L1", + "type": "raw_and_aggregated" + } + } +} diff --git a/tests/cli_autogen_input/sonic-one-table-container.yang b/tests/cli_autogen_input/sonic-one-table-container.yang new file mode 100644 index 0000000000..ca5d248ece --- /dev/null +++ b/tests/cli_autogen_input/sonic-one-table-container.yang @@ -0,0 +1,19 @@ +module sonic-one-table-container { + + yang-version 1.1; + + namespace "http://github.com/Azure/sonic-one"; + prefix one; + + container sonic-one-table-container { + + container ONE_TABLE { + + description "ONE_TABLE description"; + + leaf random { + type string; + } + } + } +} \ No newline at end of file diff --git a/tests/cli_autogen_input/sonic-vlan.yang b/tests/cli_autogen_input/sonic-vlan.yang new file mode 100644 index 0000000000..2962161ef0 --- /dev/null +++ b/tests/cli_autogen_input/sonic-vlan.yang @@ -0,0 +1,190 @@ +module sonic-vlan { + + yang-version 1.1; + + namespace "http://github.com/Azure/sonic-vlan"; + prefix vlan; + + import ietf-inet-types { + prefix inet; + } + + import sonic-types { + prefix stypes; + revision-date 2019-07-01; + } + + import sonic-extension { + prefix ext; + revision-date 2019-07-01; + } + + import sonic-port { + prefix port; + revision-date 2019-07-01; + } + + import sonic-vrf { + prefix vrf; + } + + description "VLAN yang Module for SONiC OS"; + + revision 2021-03-30 { + description "Modify the type of vrf name"; + } + + revision 2019-07-01 { + description "First Revision"; + } + + container sonic-vlan { + + container VLAN_INTERFACE { + + description "VLAN_INTERFACE part of config_db.json"; + + list VLAN_INTERFACE_LIST { + + description "VLAN INTERFACE part of config_db.json with vrf"; + + key "name"; + + leaf name { + type leafref { + path /vlan:sonic-vlan/vlan:VLAN/vlan:VLAN_LIST/vlan:name; + } + } + + leaf vrf_name { + type leafref{ + path "/vrf:sonic-vrf/vrf:VRF/vrf:VRF_LIST/vrf:name"; + } + } + } + /* end of VLAN_INTERFACE_LIST */ + + list VLAN_INTERFACE_IPPREFIX_LIST { + + key "name ip-prefix"; + + leaf name { + /* This node must be present in VLAN_INTERFACE_LIST */ + must "(current() = ../../VLAN_INTERFACE_LIST[name=current()]/name)" + { + error-message "Must condition not satisfied, Try adding Vlan: {}, Example: 'Vlan100': {}"; + } + + type leafref { + path "/vlan:sonic-vlan/vlan:VLAN/vlan:VLAN_LIST/vlan:name"; + } + } + + leaf ip-prefix { + type union { + type stypes:sonic-ip4-prefix; + type stypes:sonic-ip6-prefix; + } + } + + leaf scope { + type enumeration { + enum global; + enum local; + } + } + + leaf family { + + /* family leaf needed for backward compatibility + Both ip4 and ip6 address are string in IETF RFC 6021, + so must statement can check based on : or ., family + should be IPv4 or IPv6 according. + */ + + must "(contains(../ip-prefix, ':') and current()='IPv6') or + (contains(../ip-prefix, '.') and current()='IPv4')"; + type stypes:ip-family; + } + } + /* end of VLAN_INTERFACE_LIST */ + } + /* end of VLAN_INTERFACE container */ + + container VLAN { + + description "VLAN part of config_db.json"; + + list VLAN_LIST { + + key "name"; + + leaf name { + type string { + pattern 'Vlan([0-9]{1,3}|[1-3][0-9]{3}|[4][0][0-8][0-9]|[4][0][9][0-4])'; + } + } + + leaf vlanid { + type uint16 { + range 1..4094; + } + } + + leaf description { + type string { + length 1..255; + } + } + + leaf-list dhcp_servers { + type inet:ip-address; + } + + leaf mtu { + type uint16 { + range 1..9216; + } + } + + leaf admin_status { + type stypes:admin_status; + } + } + /* end of VLAN_LIST */ + } + /* end of container VLAN */ + + container VLAN_MEMBER { + + description "VLAN_MEMBER part of config_db.json"; + + list VLAN_MEMBER_LIST { + + key "name port"; + + leaf name { + type leafref { + path "/vlan:sonic-vlan/vlan:VLAN/vlan:VLAN_LIST/vlan:name"; + } + } + + leaf port { + /* key elements are mandatory by default */ + type leafref { + path /port:sonic-port/port:PORT/port:PORT_LIST/port:name; + } + } + + leaf tagging_mode { + mandatory true; + type stypes:vlan_tagging_mode; + } + } + /* end of list VLAN_MEMBER_LIST */ + } + /* end of container VLAN_MEMBER */ + } + /* end of container sonic-vlan */ +} +/* end of module sonic-vlan */ diff --git a/tests/cli_autogen_yang_parser_test.py b/tests/cli_autogen_yang_parser_test.py new file mode 100644 index 0000000000..fb5141aff1 --- /dev/null +++ b/tests/cli_autogen_yang_parser_test.py @@ -0,0 +1,56 @@ +import sys +import os +import pytest +import logging +# debug +import pprint + +from sonic_cli_gen.yang_parser import YangParser + +logger = logging.getLogger(__name__) + +test_path = os.path.dirname(os.path.abspath(__file__)) +config_db_path = os.path.join(test_path, 'cli_autogen_input/config_db.json') +yang_models_path = '/usr/local/yang-models' + + +class TestYangParser: + def test_one_table_container(self): + yang_model_name = 'sonic-one-table-container' + + + move_yang_model(yang_model_name) + parser = YangParser(yang_model_name = yang_model_name, + config_db_path = config_db_path, + allow_tbl_without_yang = True, + debug = False) + yang_dict = parser.parse_yang_model() + pretty_log(yang_dict) + + pass + +def move_yang_model(yang_model_name): + """ Move provided YANG model to known location for YangParser class + + Args: + yang_model_name: name of provided YANG model + """ + src_path = os.path.join(test_path, 'cli_autogen_input', yang_model_name + '.yang') + cmd = 'sudo cp {} {}'.format(src_path, yang_models_path) + os.system(cmd) + +def remove_yang_model(yang_model_name): + """ Remove YANG model from well known system location + + Args: + yang_model_name: name of provided YANG model + """ + yang_model_path = os.path.join(yang_models_path, yang_model_name + '.yang') + cmd = 'sudo rm {}'.format(yang_model_path) + os.system(cmd) + +# DEBUG function +def pretty_log(dictionary): + for line in pprint.pformat(dictionary).split('\n'): + logging.warning(line) + From 0d371de158fbac9487fbae8688a8cc1180652e69 Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Fri, 14 May 2021 19:37:23 +0000 Subject: [PATCH 100/173] Refactored _init_yang_module_and_containers() Signed-off-by: Vadym Hlushko --- sonic_cli_gen/yang_parser.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/sonic_cli_gen/yang_parser.py b/sonic_cli_gen/yang_parser.py index b36d862ee8..a114a68165 100644 --- a/sonic_cli_gen/yang_parser.py +++ b/sonic_cli_gen/yang_parser.py @@ -71,21 +71,22 @@ def _init_yang_module_and_containers(self): self.y_table_containers Raises: - KeyError: if invalid YANG model provided - KeyError: if YANG models is NOT exist + KeyError: if YANG model is invalid or NOT exist """ self._find_index_of_yang_model() - if self.idx_yJson is not None: - self.y_module = self.conf_mgmt.sy.yJson[self.idx_yJson]['module'] - if self.y_module.get('container') is not None: - self.y_top_level_container = self.y_module['container'] - self.y_table_containers = self.y_top_level_container['container'] - else: - raise KeyError('YANG model {} does NOT have "container" element'.format(self.yang_model_name)) - else: + if self.idx_yJson is None: raise KeyError('YANG model {} is NOT exist'.format(self.yang_model_name)) + self.y_module = self.conf_mgmt.sy.yJson[self.idx_yJson]['module'] + + if self.y_module.get('container') is None: + raise KeyError('YANG model {} does NOT have "top level container" element'.format(self.yang_model_name)) + self.y_top_level_container = self.y_module.get('container') + + if self.y_top_level_container.get('container') is None: + raise KeyError('YANG model {} does NOT have "container" element after "top level container"'.format(self.yang_model_name)) + self.y_table_containers = self.y_top_level_container.get('container') def _find_index_of_yang_model(self): """ Find index of provided YANG model inside yJson object From 1593039dadf59f1df8c57c1b153bbc778f11d205 Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Fri, 14 May 2021 20:32:16 +0000 Subject: [PATCH 101/173] added 2 test cases Signed-off-by: Vadym Hlushko --- sonic_cli_gen/yang_parser.py | 15 ++++--- .../cli_autogen_input/assert_dictionaries.py | 40 +++++++++++++++++++ .../sonic-many-table-containers.yang | 20 ++++++++++ .../sonic-one-table-container.yang | 12 ++---- tests/cli_autogen_yang_parser_test.py | 30 +++++++++----- 5 files changed, 95 insertions(+), 22 deletions(-) create mode 100644 tests/cli_autogen_input/assert_dictionaries.py create mode 100644 tests/cli_autogen_input/sonic-many-table-containers.yang diff --git a/sonic_cli_gen/yang_parser.py b/sonic_cli_gen/yang_parser.py index a114a68165..32e2b1a7f0 100644 --- a/sonic_cli_gen/yang_parser.py +++ b/sonic_cli_gen/yang_parser.py @@ -81,16 +81,21 @@ def _init_yang_module_and_containers(self): self.y_module = self.conf_mgmt.sy.yJson[self.idx_yJson]['module'] if self.y_module.get('container') is None: - raise KeyError('YANG model {} does NOT have "top level container" element'.format(self.yang_model_name)) + raise KeyError('YANG model {} does NOT have "top level container" element \ + Please follow the SONiC YANG model guidelines: \ + https://github.com/Azure/SONiC/blob/master/doc/mgmt/SONiC_YANG_Model_Guidelines.md'.format(self.yang_model_name)) self.y_top_level_container = self.y_module.get('container') if self.y_top_level_container.get('container') is None: - raise KeyError('YANG model {} does NOT have "container" element after "top level container"'.format(self.yang_model_name)) + raise KeyError('YANG model {} does NOT have "container" element after "top level container" \ + Please follow the SONiC YANG model guidelines: \ + https://github.com/Azure/SONiC/blob/master/doc/mgmt/SONiC_YANG_Model_Guidelines.md'.format(self.yang_model_name)) self.y_table_containers = self.y_top_level_container.get('container') def _find_index_of_yang_model(self): - """ Find index of provided YANG model inside yJson object + """ Find index of provided YANG model inside yJson object, and save it to self.idx_yJson variable + yJson object contain all yang-models parsed from directory - /usr/local/yang-models """ for i in range(len(self.conf_mgmt.sy.yJson)): @@ -99,7 +104,7 @@ def _find_index_of_yang_model(self): def parse_yang_model(self) -> dict: """ Parse proviced YANG model - and save output to self.yang_2_dict obj + and save output to self.yang_2_dict object Returns: dictionary - parsed YANG model in dictionary format @@ -108,7 +113,7 @@ def parse_yang_model(self) -> dict: self._init_yang_module_and_containers() self.yang_2_dict['tables'] = list() - # determine how many (1 or couple) containers YANG model have after 'top level' container + # determine how many (1 or couple) containers a YANG model have after 'top level' container # 'table' container it is a container that goes after 'top level' container if isinstance(self.y_table_containers, list): for tbl_cont in self.y_table_containers: diff --git a/tests/cli_autogen_input/assert_dictionaries.py b/tests/cli_autogen_input/assert_dictionaries.py new file mode 100644 index 0000000000..623233e7ee --- /dev/null +++ b/tests/cli_autogen_input/assert_dictionaries.py @@ -0,0 +1,40 @@ +""" +Module holding correct dictionaries for test YANG models +""" + +one_table_container = { + "tables":[ + { + "description":"FIRST_TABLE description", + "name":"FIRST_TABLE", + "static-objects":[ + { + + } + ] + } + ] +} + +many_table_containers = { + "tables":[ + { + "description":"FIRST_TABLE description", + "name":"FIRST_TABLE", + "static-objects":[ + { + + } + ] + }, + { + "description":"SECOND_TABLE description", + "name":"SECOND_TABLE", + "static-objects":[ + { + + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/cli_autogen_input/sonic-many-table-containers.yang b/tests/cli_autogen_input/sonic-many-table-containers.yang new file mode 100644 index 0000000000..41a7d3c288 --- /dev/null +++ b/tests/cli_autogen_input/sonic-many-table-containers.yang @@ -0,0 +1,20 @@ +module sonic-many-table-containers { + + yang-version 1.1; + + namespace "http://github.com/Azure/sonic-many-table-cont"; + prefix many-table-cont; + + container sonic-many-table-containers { + + container FIRST_TABLE { + + description "FIRST_TABLE description"; + } + + container SECOND_TABLE { + + description "SECOND_TABLE description"; + } + } +} diff --git a/tests/cli_autogen_input/sonic-one-table-container.yang b/tests/cli_autogen_input/sonic-one-table-container.yang index ca5d248ece..b04afa7ed9 100644 --- a/tests/cli_autogen_input/sonic-one-table-container.yang +++ b/tests/cli_autogen_input/sonic-one-table-container.yang @@ -2,18 +2,14 @@ module sonic-one-table-container { yang-version 1.1; - namespace "http://github.com/Azure/sonic-one"; - prefix one; + namespace "http://github.com/Azure/sonic-one-table"; + prefix one-table; container sonic-one-table-container { - container ONE_TABLE { + container FIRST_TABLE { - description "ONE_TABLE description"; - - leaf random { - type string; - } + description "FIRST_TABLE description"; } } } \ No newline at end of file diff --git a/tests/cli_autogen_yang_parser_test.py b/tests/cli_autogen_yang_parser_test.py index fb5141aff1..e9010c4877 100644 --- a/tests/cli_autogen_yang_parser_test.py +++ b/tests/cli_autogen_yang_parser_test.py @@ -6,28 +6,40 @@ import pprint from sonic_cli_gen.yang_parser import YangParser +from .cli_autogen_input import assert_dictionaries + logger = logging.getLogger(__name__) test_path = os.path.dirname(os.path.abspath(__file__)) -config_db_path = os.path.join(test_path, 'cli_autogen_input/config_db.json') yang_models_path = '/usr/local/yang-models' class TestYangParser: + + def test_one_table_container(self): yang_model_name = 'sonic-one-table-container' + template('sonic-one-table-container', assert_dictionaries.one_table_container) + def test_many_table_containers(self): + yang_model_name = 'sonic-many-table-containers' + template('sonic-many-table-containers', assert_dictionaries.many_table_containers) + +def template(yang_model_name, correct_dict): + config_db_path = os.path.join(test_path, 'cli_autogen_input/config_db.json') + move_yang_model(yang_model_name) + parser = YangParser(yang_model_name = yang_model_name, + config_db_path = config_db_path, + allow_tbl_without_yang = True, + debug = False) + yang_dict = parser.parse_yang_model() + # debug + pretty_log(yang_dict) - move_yang_model(yang_model_name) - parser = YangParser(yang_model_name = yang_model_name, - config_db_path = config_db_path, - allow_tbl_without_yang = True, - debug = False) - yang_dict = parser.parse_yang_model() - pretty_log(yang_dict) + assert yang_dict == correct_dict - pass + remove_yang_model(yang_model_name) def move_yang_model(yang_model_name): """ Move provided YANG model to known location for YangParser class From 867d9c19b5d8a5a84ba9511888d73e38e5eabc05 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Mon, 17 May 2021 12:36:14 +0300 Subject: [PATCH 102/173] [sonic_package_manager] flush once finished saving docker image into temporary file Since installation happens inside context manager "with tempfile.NamedTemporaryFile()" it may be that depending on file size/number of writes the docker image isn't fully written to the disk. This fixes change fixes the issue by adding a flush. Signed-off-by: Stepan Blyschak --- sonic_package_manager/manager.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sonic_package_manager/manager.py b/sonic_package_manager/manager.py index 24d505eb0c..b2245acc01 100644 --- a/sonic_package_manager/manager.py +++ b/sonic_package_manager/manager.py @@ -726,6 +726,7 @@ def migrate_package(old_package_entry, with tempfile.NamedTemporaryFile('wb') as file: for chunk in image.save(named=True): file.write(chunk) + file.flush() self.install(tarball=file.name) else: From 3f00843d43f592d021e08914271c1519c166fcdc Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Mon, 17 May 2021 13:16:13 +0300 Subject: [PATCH 103/173] move cli autogen to ServiceCreator Signed-off-by: Stepan Blyschak --- sonic_package_manager/manager.py | 31 ++------ .../service_creator/__init__.py | 1 + .../service_creator/creator.py | 72 ++++++++++++++++++- tests/sonic_package_manager/conftest.py | 8 +-- .../test_service_creator.py | 45 +++++++++++- 5 files changed, 120 insertions(+), 37 deletions(-) diff --git a/sonic_package_manager/manager.py b/sonic_package_manager/manager.py index b2245acc01..4838374f3a 100644 --- a/sonic_package_manager/manager.py +++ b/sonic_package_manager/manager.py @@ -43,6 +43,7 @@ from sonic_package_manager.progress import ProgressManager from sonic_package_manager.reference import PackageReference from sonic_package_manager.registry import RegistryResolver +from sonic_package_manager.service_creator import SONIC_CLI_COMMANDS from sonic_package_manager.service_creator.creator import ( ServiceCreator, run_command @@ -64,9 +65,6 @@ ) -SONIC_CLI_COMMANDS = ('show', 'config', 'clear') - - @contextlib.contextmanager def failure_ignore(ignore: bool): """ Ignores failures based on parameter passed. """ @@ -310,7 +308,6 @@ def __init__(self, database: PackageDatabase, metadata_resolver: MetadataResolver, service_creator: ServiceCreator, - cli_generator: CliGenerator, device_information: Any, lock: filelock.FileLock): """ Initialize PackageManager. """ @@ -321,7 +318,6 @@ def __init__(self, self.database = database self.metadata_resolver = metadata_resolver self.service_creator = service_creator - self.cli_generator = cli_generator self.feature_registry = service_creator.feature_registry self.is_multi_npu = device_information.is_multi_npu() self.num_npus = device_information.get_num_npus() @@ -978,12 +974,10 @@ def _systemctl_action(self, package: Package, action: str): def _install_cli_plugins(self, package: Package): for command in SONIC_CLI_COMMANDS: self._install_cli_plugin(package, command) - self._install_autogen_cli(package, command) def _uninstall_cli_plugins(self, package: Package): for command in SONIC_CLI_COMMANDS: self._uninstall_cli_plugin(package, command) - self._uninstall_autogen_cli(package, command) def _install_cli_plugin(self, package: Package, command: str): image_plugin_path = package.manifest['cli'][command] @@ -1000,23 +994,6 @@ def _uninstall_cli_plugin(self, package: Package, command: str): if os.path.exists(host_plugin_path): os.remove(host_plugin_path) - def _install_autogen_cli(self, package: Package, command: str): - if package.metadata.yang_module_text is None: - return - if not package.manifest['cli'][f'auto-generate-{command}']: - return - cfg_mgmt = self.service_creator.cfg_mgmt - module_name = cfg_mgmt.get_module_name(package.metadata.yang_module_text) - - plugin_path = get_cli_plugin_path(package, command, 'auto') - with open(plugin_path, 'w') as out: - self.cli_generator.generate_cli_plugin(module_name, command, out) - - def _uninstall_autogen_cli(self, package: Package, command: str): - plugin_path = get_cli_plugin_path(package, command, 'auto') - if os.path.exists(plugin_path): - os.remove(plugin_path) - @staticmethod def get_manager() -> 'PackageManager': """ Creates and returns PackageManager instance. @@ -1032,13 +1009,15 @@ def get_manager() -> 'PackageManager': cli_generator = CliGenerator() sonic_db = SonicDB() feature_registry = FeatureRegistry(sonic_db) - service_creator = ServiceCreator(feature_registry, sonic_db, cfg_mgmt) + service_creator = ServiceCreator(feature_registry, + sonic_db, + cfg_mgmt, + cli_generator) return PackageManager(docker_api, registry_resolver, PackageDatabase.from_file(), metadata_resolver, service_creator, - cli_generator, device_info, filelock.FileLock(PACKAGE_MANAGER_LOCK_FILE, timeout=0)) diff --git a/sonic_package_manager/service_creator/__init__.py b/sonic_package_manager/service_creator/__init__.py index e2af81ceb5..b0f4a24086 100644 --- a/sonic_package_manager/service_creator/__init__.py +++ b/sonic_package_manager/service_creator/__init__.py @@ -1,3 +1,4 @@ #!/usr/bin/env python ETC_SONIC_PATH = '/etc/sonic' +SONIC_CLI_COMMANDS = ('show', 'config', 'clear') diff --git a/sonic_package_manager/service_creator/creator.py b/sonic_package_manager/service_creator/creator.py index a23769ef58..5044b6fb2e 100644 --- a/sonic_package_manager/service_creator/creator.py +++ b/sonic_package_manager/service_creator/creator.py @@ -4,7 +4,6 @@ import os import stat import subprocess -import pkgutil from collections import defaultdict from typing import Dict, Type @@ -14,9 +13,14 @@ from toposort import toposort_flatten, CircularDependencyError from utilities_common.general import load_module_from_source +from sonic_cli_gen.generator import CliGenerator + from sonic_package_manager.logger import log from sonic_package_manager.package import Package -from sonic_package_manager.service_creator import ETC_SONIC_PATH +from sonic_package_manager.service_creator import ( + ETC_SONIC_PATH, + SONIC_CLI_COMMANDS, +) from sonic_package_manager.service_creator.feature import FeatureRegistry from sonic_package_manager.service_creator.sonic_db import SonicDB from sonic_package_manager.service_creator.utils import in_chroot @@ -122,6 +126,7 @@ class ServiceCreator: def __init__(self, feature_registry: FeatureRegistry, sonic_db: Type[SonicDB], + cli_gen: CliGenerator, cfg_mgmt: ConfigMgmt): """ Initialize ServiceCreator with: @@ -133,6 +138,7 @@ def __init__(self, self.feature_registry = feature_registry self.sonic_db = sonic_db + self.cli_gen = cli_gen self.cfg_mgmt = cfg_mgmt def create(self, @@ -161,6 +167,7 @@ def create(self, self.generate_service_reconciliation_file(package) self.install_yang_module(package) self.set_initial_config(package) + self.install_autogen_cli_all(package) self._post_operation_hook() if register_feature: @@ -196,6 +203,7 @@ def remove(self, if deregister_feature and not keep_config: self.remove_config(package) + self.uninstall_autogen_cli_all(package) self.uninstall_yang_module(package) self._post_operation_hook() @@ -557,6 +565,66 @@ def uninstall_yang_module(self, package: Package): module_name = self.cfg_mgmt.get_module_name(package.metadata.yang_module_text) self.cfg_mgmt.remove_module(module_name) + def install_autogen_cli_all(self, package: Package): + """ Install autogenerated CLI plugins for package. + + Args: + package: Package + Returns: + None + """ + + for command in SONIC_CLI_COMMANDS: + self.install_autogen_cli(package, command) + + def uninstall_autogen_cli_all(self, package: Package): + """ Remove autogenerated CLI plugins for package. + + Args: + package: Package + Returns: + None + """ + + for command in SONIC_CLI_COMMANDS: + self.uninstall_autogen_cli(package, command) + + def install_autogen_cli(self, package: Package, command: str): + """ Install autogenerated CLI plugins for package for particular command. + + Args: + package: Package. + command: Name of command to generate CLI for. + Returns: + None + """ + + if package.metadata.yang_module_text is None: + return + if not package.manifest['cli'][f'auto-generate-{command}']: + return + module_name = self.cfg_mgmt.get_module_name(package.metadata.yang_module_text) + self.cli_gen.generate_cli_plugin(module_name, command) + log.debug(f'{command} command line interface autogenerated for {module_name}') + + def uninstall_autogen_cli(self, package: Package, command: str): + """ Uninstall autogenerated CLI plugins for package for particular command. + + Args: + package: Package. + command: Name of command to remove CLI. + Returns: + None + """ + + if package.metadata.yang_module_text is None: + return + if not package.manifest['cli'][f'auto-generate-{command}']: + return + module_name = self.cfg_mgmt.get_module_name(package.metadata.yang_module_text) + self.cli_gen.remove_cli_plugin(module_name, command) + log.debug(f'{command} command line interface removed for {module_name}') + def _post_operation_hook(self): """ Common operations executed after service is created/removed. """ diff --git a/tests/sonic_package_manager/conftest.py b/tests/sonic_package_manager/conftest.py index e5bdebb4c4..1ec067657c 100644 --- a/tests/sonic_package_manager/conftest.py +++ b/tests/sonic_package_manager/conftest.py @@ -418,24 +418,20 @@ def sonic_fs(fs): @pytest.fixture(autouse=True) def patch_pkgutil(): - yield - # with mock.patch('pkgutil.get_loader') as loader: - # yield loader + with mock.patch('pkgutil.get_loader') as loader: + yield loader @pytest.fixture def package_manager(mock_docker_api, mock_registry_resolver, mock_service_creator, - mock_cli_gen, - mock_config_mgmt, fake_metadata_resolver, fake_db, fake_device_info): yield PackageManager(mock_docker_api, mock_registry_resolver, fake_db, fake_metadata_resolver, mock_service_creator, - mock_cli_gen, fake_device_info, MagicMock()) diff --git a/tests/sonic_package_manager/test_service_creator.py b/tests/sonic_package_manager/test_service_creator.py index c24dcea157..f6909ce6ba 100644 --- a/tests/sonic_package_manager/test_service_creator.py +++ b/tests/sonic_package_manager/test_service_creator.py @@ -1,7 +1,7 @@ #!/usr/bin/env python import os -from unittest.mock import Mock, MagicMock +from unittest.mock import Mock, call import pytest @@ -60,8 +60,16 @@ def manifest(): @pytest.fixture() -def service_creator(mock_feature_registry, mock_sonic_db, mock_config_mgmt): - yield ServiceCreator(mock_feature_registry, mock_sonic_db, mock_config_mgmt) +def service_creator(mock_feature_registry, + mock_sonic_db, + mock_cli_gen, + mock_config_mgmt): + yield ServiceCreator( + mock_feature_registry, + mock_sonic_db, + mock_cli_gen, + mock_config_mgmt + ) def test_service_creator(sonic_fs, manifest, service_creator, package_manager): @@ -166,6 +174,37 @@ def test_service_creator_yang(sonic_fs, manifest, mock_sonic_db, mock_config_mgmt.remove_module.assert_called_with(test_yang_module) +def test_service_creator_autocli(sonic_fs, manifest, mock_cli_gen, + mock_config_mgmt, service_creator): + test_yang = 'TEST YANG' + test_yang_module = 'sonic-test' + + manifest['cli']['auto-generate-show'] = True + manifest['cli']['auto-generate-config'] = True + + entry = PackageEntry('test', 'azure/sonic-test') + package = Package(entry, Metadata(manifest, yang_module_text=test_yang)) + mock_config_mgmt.get_module_name = Mock(return_value=test_yang_module) + service_creator.create(package) + + mock_cli_gen.generate_cli_plugin.assert_has_calls( + [ + call(test_yang_module, 'show'), + call(test_yang_module, 'config'), + ], + any_order=True + ) + + service_creator.remove(package) + mock_cli_gen.remove_cli_plugin.assert_has_calls( + [ + call(test_yang_module, 'show'), + call(test_yang_module, 'config'), + ], + any_order=True + ) + + def test_feature_registration(mock_sonic_db, manifest): mock_connector = Mock() mock_connector.get_entry = Mock(return_value={}) From b3b0a8e4a61bf56e11eaf0c91f2059be12f2d928 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Mon, 17 May 2021 13:16:31 +0300 Subject: [PATCH 104/173] Revert "[sonic-cli-gen]" This reverts commit 65b125c495af878a9092ea2f37867f8a6e045e08. --- .../templates/sonic-cli-gen/config.py.j2 | 2 +- sonic_cli_gen/generator.py | 19 ++++++++----------- sonic_cli_gen/main.py | 8 ++++---- 3 files changed, 13 insertions(+), 16 deletions(-) diff --git a/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 b/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 index 89580539f5..ac19835cd1 100644 --- a/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 +++ b/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 @@ -396,7 +396,7 @@ def {{ group }}_delete(db, {{ pythonize(object["keys"]) }}): {# Generate another nesting group in case table holds two types of objects #} {% if table["dynamic-objects"]|length > 1 %} {% set group = table.name + "_" + object.name %} -@{{ table.name }}.group(name="{{ cli_name(object.name|replace(table.name + "_", "", 1)) }}", +@{{ table.name }}.group(name="{{ cli_name(object.name) }}", cls=clicommon.AliasedGroup) def {{ group }}(): """ {{ object.description }} """ diff --git a/sonic_cli_gen/generator.py b/sonic_cli_gen/generator.py index 199149ff73..0a02a104bc 100644 --- a/sonic_cli_gen/generator.py +++ b/sonic_cli_gen/generator.py @@ -12,25 +12,22 @@ class CliGenerator: show, sonic-clear CLI plugins """ - def __init__(self): + def __init__(self, + yang_model): """ Initialize PackageManager. """ + self.yang_model_name = yang_model self.loader = jinja2.FileSystemLoader(['/usr/share/sonic/templates/sonic-cli-gen/']) self.env = jinja2.Environment(loader=self.loader) - def generate_cli_plugin(self, yang_model_name, cli_group, output_stream=None): + def generate_cli_plugin(self, cli_group, plugin_name): """ Generate CLI plugin. """ - parser = YangParser(yang_model_name) + parser = YangParser(self.yang_model_name) yang_dict = parser.parse_yang_model() + plugin_path = get_cli_plugin_path(cli_group, plugin_name + '_yang.py') template = self.env.get_template(cli_group + '.py.j2') - - if output_stream is None: - plugin_name = yang_module_name.replace('-', '_') - plugin_path = get_cli_plugin_path(cli_group, plugin_name + '_auto.py') - with open(plugin_path, 'w') as output_stream: - output_stream.write(template.render(yang_dict)) - else: - output_stream.write(template.render(yang_dict)) + with open(plugin_path, 'w') as plugin_py: + plugin_py.write(template.render(yang_dict)) def get_cli_plugin_path(command, plugin_name): pkg_loader = pkgutil.get_loader(f'{command}.plugins') diff --git a/sonic_cli_gen/main.py b/sonic_cli_gen/main.py index 6e1b1874e8..8cf748e394 100644 --- a/sonic_cli_gen/main.py +++ b/sonic_cli_gen/main.py @@ -14,16 +14,16 @@ def cli(ctx): @click.pass_context def generate_config(ctx, yang_model_name): """ Generate CLI plugin (click) for 'config' CLI group. """ - gen = CliGenerator() - gen.generate_cli_plugin(yang_model_name, cli_group='config', plugin_name=yang_model_name) + gen = CliGenerator(yang_model_name) + gen.generate_cli_plugin(cli_group='config', plugin_name=yang_model_name) @cli.command() @click.argument('yang_model_name') @click.pass_context def generate_show(ctx, yang_model_name): """ Generate CLI plugin (click) for 'show' CLI group. """ - gen = CliGenerator() - gen.generate_cli_plugin(yang_model_name, cli_group='show', plugin_name=yang_model_name) + gen = CliGenerator(yang_model_name) + gen.generate_cli_plugin(cli_group='show', plugin_name=yang_model_name) if __name__ == '__main__': cli() \ No newline at end of file From def7607a5553089d638a96a16668a3178a8955a4 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Mon, 17 May 2021 13:16:42 +0300 Subject: [PATCH 105/173] Revert "[sonic-cli-gen]" This reverts commit 134b21e8f529b550834d54ea502341c644a1cf23. --- sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 b/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 index ea82b9761f..58e7dfd2e0 100644 --- a/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 +++ b/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 @@ -77,8 +77,7 @@ def {{ name }}(db): body = [] table = db.cfgdb.get_table("{{ table.name }}") - for key in natsort.natsorted(table): - entry = table[key] + for key, entry in natsort.natsorted(table).items(): if not isinstance(key, tuple): key = (key,) From ada4ca42e3d20417003d31eada2293545fb5bbb5 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Mon, 17 May 2021 13:29:20 +0300 Subject: [PATCH 106/173] [sonic-cli-gen] add remove plugin method Signed-off-by: Stepan Blyschak --- sonic_cli_gen/generator.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/sonic_cli_gen/generator.py b/sonic_cli_gen/generator.py index 6214bbe9c5..e8d0421f71 100644 --- a/sonic_cli_gen/generator.py +++ b/sonic_cli_gen/generator.py @@ -27,6 +27,12 @@ def generate_cli_plugin(self, cli_group, plugin_name): template = self.env.get_template(cli_group + '.py.j2') with open(plugin_path, 'w') as plugin_py: plugin_py.write(template.render(yang_dict)) + + def remove_cli_plugin(self, cli_group, plugin_name): + plugin_path = get_cli_plugin_path(cli_group, plugin_name + '_yang.py') + if os.path.exists(plugin_path): + os.remove(plugin_path) + def get_cli_plugin_path(command, plugin_name): pkg_loader = pkgutil.get_loader(f'{command}.plugins') From 661aefb080f60c2d28daa025b1cb8f6b80c16a56 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Mon, 17 May 2021 13:30:01 +0300 Subject: [PATCH 107/173] align method signature with CliGenerator Signed-off-by: Stepan Blyschak --- sonic_package_manager/service_creator/creator.py | 4 ++-- tests/sonic_package_manager/test_service_creator.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/sonic_package_manager/service_creator/creator.py b/sonic_package_manager/service_creator/creator.py index 5044b6fb2e..21ac11a846 100644 --- a/sonic_package_manager/service_creator/creator.py +++ b/sonic_package_manager/service_creator/creator.py @@ -604,7 +604,7 @@ def install_autogen_cli(self, package: Package, command: str): if not package.manifest['cli'][f'auto-generate-{command}']: return module_name = self.cfg_mgmt.get_module_name(package.metadata.yang_module_text) - self.cli_gen.generate_cli_plugin(module_name, command) + self.cli_gen.generate_cli_plugin(command, module_name) log.debug(f'{command} command line interface autogenerated for {module_name}') def uninstall_autogen_cli(self, package: Package, command: str): @@ -622,7 +622,7 @@ def uninstall_autogen_cli(self, package: Package, command: str): if not package.manifest['cli'][f'auto-generate-{command}']: return module_name = self.cfg_mgmt.get_module_name(package.metadata.yang_module_text) - self.cli_gen.remove_cli_plugin(module_name, command) + self.cli_gen.remove_cli_plugin(command, module_name) log.debug(f'{command} command line interface removed for {module_name}') def _post_operation_hook(self): diff --git a/tests/sonic_package_manager/test_service_creator.py b/tests/sonic_package_manager/test_service_creator.py index f6909ce6ba..9bfac219d4 100644 --- a/tests/sonic_package_manager/test_service_creator.py +++ b/tests/sonic_package_manager/test_service_creator.py @@ -189,8 +189,8 @@ def test_service_creator_autocli(sonic_fs, manifest, mock_cli_gen, mock_cli_gen.generate_cli_plugin.assert_has_calls( [ - call(test_yang_module, 'show'), - call(test_yang_module, 'config'), + call('show', test_yang_module), + call('config', test_yang_module), ], any_order=True ) @@ -198,8 +198,8 @@ def test_service_creator_autocli(sonic_fs, manifest, mock_cli_gen, service_creator.remove(package) mock_cli_gen.remove_cli_plugin.assert_has_calls( [ - call(test_yang_module, 'show'), - call(test_yang_module, 'config'), + call('show', test_yang_module), + call('config', test_yang_module), ], any_order=True ) From ad7ee336bce60eb3100545a1064e599a4d44d73b Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Mon, 17 May 2021 13:48:32 +0300 Subject: [PATCH 108/173] [sonic-cli-gen] put autogenerated plugins into plugins.auto Signed-off-by: Stepan Blyschak --- clear/plugins/auto/__init__.py | 0 config/plugins/auto/__init__.py | 0 setup.py | 3 +++ show/plugins/auto/__init__.py | 0 sonic_cli_gen/generator.py | 2 +- utilities_common/util_base.py | 1 + 6 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 clear/plugins/auto/__init__.py create mode 100644 config/plugins/auto/__init__.py create mode 100644 show/plugins/auto/__init__.py diff --git a/clear/plugins/auto/__init__.py b/clear/plugins/auto/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/config/plugins/auto/__init__.py b/config/plugins/auto/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/setup.py b/setup.py index 876eadb425..b1afdcd906 100644 --- a/setup.py +++ b/setup.py @@ -23,8 +23,10 @@ 'acl_loader', 'clear', 'clear.plugins', + 'clear.plugins.auto', 'config', 'config.plugins', + 'config.plugins.auto', 'connect', 'consutil', 'counterpoll', @@ -46,6 +48,7 @@ 'show', 'show.interfaces', 'show.plugins', + 'show.plugins.auto', 'sonic_installer', 'sonic_installer.bootloader', 'sonic_package_manager', diff --git a/show/plugins/auto/__init__.py b/show/plugins/auto/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/sonic_cli_gen/generator.py b/sonic_cli_gen/generator.py index e8d0421f71..adb6ac5a96 100644 --- a/sonic_cli_gen/generator.py +++ b/sonic_cli_gen/generator.py @@ -35,7 +35,7 @@ def remove_cli_plugin(self, cli_group, plugin_name): def get_cli_plugin_path(command, plugin_name): - pkg_loader = pkgutil.get_loader(f'{command}.plugins') + pkg_loader = pkgutil.get_loader(f'{command}.plugins.auto') if pkg_loader is None: raise PackageManagerError(f'Failed to get plugins path for {command} CLI') plugins_pkg_path = os.path.dirname(pkg_loader.path) diff --git a/utilities_common/util_base.py b/utilities_common/util_base.py index ff5570735c..9bea158b59 100644 --- a/utilities_common/util_base.py +++ b/utilities_common/util_base.py @@ -24,6 +24,7 @@ def iter_namespace(ns_pkg): for _, module_name, ispkg in iter_namespace(plugins_namespace): if ispkg: + yield from self.load_plugins(importlib.import_module(module_name)) continue log.log_debug('importing plugin: {}'.format(module_name)) try: From 087c5273026704f845656ca35074d5149e70e8c6 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Mon, 17 May 2021 15:38:45 +0300 Subject: [PATCH 109/173] fix wrong parameters order when creating ServiceCreator Signed-off-by: Stepan Blyschak --- sonic_package_manager/manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sonic_package_manager/manager.py b/sonic_package_manager/manager.py index 4838374f3a..2aef03e1fa 100644 --- a/sonic_package_manager/manager.py +++ b/sonic_package_manager/manager.py @@ -1011,8 +1011,8 @@ def get_manager() -> 'PackageManager': feature_registry = FeatureRegistry(sonic_db) service_creator = ServiceCreator(feature_registry, sonic_db, - cfg_mgmt, - cli_generator) + cli_generator, + cfg_mgmt) return PackageManager(docker_api, registry_resolver, From 471a1280eb64308d8c2ce0baa475205d31585adf Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Mon, 17 May 2021 15:45:21 +0300 Subject: [PATCH 110/173] [sonic-cli-gen] fix show.py.j2 template Signed-off-by: Stepan Blyschak --- sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 b/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 index 58e7dfd2e0..ea82b9761f 100644 --- a/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 +++ b/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 @@ -77,7 +77,8 @@ def {{ name }}(db): body = [] table = db.cfgdb.get_table("{{ table.name }}") - for key, entry in natsort.natsorted(table).items(): + for key in natsort.natsorted(table): + entry = table[key] if not isinstance(key, tuple): key = (key,) From 9c0aa8b2546720e920ea1c834ebe9300a3bdde3e Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Mon, 17 May 2021 15:49:50 +0300 Subject: [PATCH 111/173] add an option to keep features config Signed-off-by: Stepan Blyschak --- sonic_package_manager/main.py | 10 ++++++++-- sonic_package_manager/manager.py | 7 +++++-- sonic_package_manager/service_creator/creator.py | 4 ++-- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/sonic_package_manager/main.py b/sonic_package_manager/main.py index c0589ae5b5..9034da7fd7 100644 --- a/sonic_package_manager/main.py +++ b/sonic_package_manager/main.py @@ -414,10 +414,11 @@ def reset(ctx, name, force, yes, skip_host_plugins): @cli.command() @add_options(PACKAGE_COMMON_OPERATION_OPTIONS) +@click.option('--keep-config', is_flag=True, help='Keep features configuration in CONFIG DB.') @click.argument('name') @click.pass_context @root_privileges_required -def uninstall(ctx, name, force, yes): +def uninstall(ctx, name, force, yes, keep_config): """ Uninstall package. """ manager: PackageManager = ctx.obj @@ -425,9 +426,14 @@ def uninstall(ctx, name, force, yes): if not yes and not force: click.confirm(f'Package {name} is going to be uninstalled, ' f'continue?', abort=True, show_default=True) + + uninstall_opts = { + 'force': force, + 'keep_config': keep_config, + } try: - manager.uninstall(name, force) + manager.uninstall(name, **uninstall_opts) except Exception as err: exit_cli(f'Failed to uninstall package {name}: {err}', fg='red') except KeyboardInterrupt: diff --git a/sonic_package_manager/manager.py b/sonic_package_manager/manager.py index 2aef03e1fa..4ceff4abcb 100644 --- a/sonic_package_manager/manager.py +++ b/sonic_package_manager/manager.py @@ -457,13 +457,16 @@ def install_from_source(self, @under_lock @opt_check - def uninstall(self, name: str, force=False): + def uninstall(self, name: str, + force: bool = False, + keep_config: bool = False): """ Uninstall SONiC Package referenced by name. The uninstallation can be forced if force argument is True. Args: name: SONiC Package name. force: Force the installation. + keep_config: Keep feature configuration in databases. Raises: PackageManagerError """ @@ -493,7 +496,7 @@ def uninstall(self, name: str, force=False): try: self._uninstall_cli_plugins(package) - self.service_creator.remove(package) + self.service_creator.remove(package, keep_config=keep_config) self.service_creator.generate_shutdown_sequence_files( self._get_installed_packages_except(package) ) diff --git a/sonic_package_manager/service_creator/creator.py b/sonic_package_manager/service_creator/creator.py index 21ac11a846..28b47e5a54 100644 --- a/sonic_package_manager/service_creator/creator.py +++ b/sonic_package_manager/service_creator/creator.py @@ -104,7 +104,7 @@ def run_command(command: str): Args: command: String command to execute as bash script Raises: - PackageManagerError: Raised when the command return code + ServiceCreatorError: Raised when the command return code is not 0. """ @@ -173,7 +173,7 @@ def create(self, if register_feature: self.feature_registry.register(package.manifest, state, owner) except (Exception, KeyboardInterrupt): - self.remove(package, register_feature) + self.remove(package, deregister_feature=register_feature) raise def remove(self, From c3350d36f552ce1d62478759c053bdb4a8582820 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Mon, 17 May 2021 15:55:04 +0300 Subject: [PATCH 112/173] fix remove() got an unexpected keyword argumet register_feature Signed-off-by: Stepan Blyschak --- sonic_package_manager/manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sonic_package_manager/manager.py b/sonic_package_manager/manager.py index 4ceff4abcb..53884d2560 100644 --- a/sonic_package_manager/manager.py +++ b/sonic_package_manager/manager.py @@ -587,7 +587,7 @@ def upgrade_from_source(self, } service_remove_opts = { - 'register_feature': False, + 'deregister_feature': False, } try: From 9004c1cb55c70451efef83c472cbb38dee0a7c60 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Mon, 17 May 2021 13:48:32 +0300 Subject: [PATCH 113/173] [sonic-cli-gen] put autogenerated plugins into plugins.auto Signed-off-by: Stepan Blyschak --- clear/plugins/auto/__init__.py | 0 config/plugins/auto/__init__.py | 0 setup.py | 3 +++ show/plugins/auto/__init__.py | 0 sonic_cli_gen/generator.py | 2 +- utilities_common/util_base.py | 1 + 6 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 clear/plugins/auto/__init__.py create mode 100644 config/plugins/auto/__init__.py create mode 100644 show/plugins/auto/__init__.py diff --git a/clear/plugins/auto/__init__.py b/clear/plugins/auto/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/config/plugins/auto/__init__.py b/config/plugins/auto/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/setup.py b/setup.py index 8639a52412..9e3f30e3ad 100644 --- a/setup.py +++ b/setup.py @@ -23,8 +23,10 @@ 'acl_loader', 'clear', 'clear.plugins', + 'clear.plugins.auto', 'config', 'config.plugins', + 'config.plugins.auto', 'connect', 'consutil', 'counterpoll', @@ -46,6 +48,7 @@ 'show', 'show.interfaces', 'show.plugins', + 'show.plugins.auto', 'sonic_installer', 'sonic_installer.bootloader', 'sonic_package_manager', diff --git a/show/plugins/auto/__init__.py b/show/plugins/auto/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/sonic_cli_gen/generator.py b/sonic_cli_gen/generator.py index 6214bbe9c5..f64395c18b 100644 --- a/sonic_cli_gen/generator.py +++ b/sonic_cli_gen/generator.py @@ -29,7 +29,7 @@ def generate_cli_plugin(self, cli_group, plugin_name): plugin_py.write(template.render(yang_dict)) def get_cli_plugin_path(command, plugin_name): - pkg_loader = pkgutil.get_loader(f'{command}.plugins') + pkg_loader = pkgutil.get_loader(f'{command}.plugins.auto') if pkg_loader is None: raise PackageManagerError(f'Failed to get plugins path for {command} CLI') plugins_pkg_path = os.path.dirname(pkg_loader.path) diff --git a/utilities_common/util_base.py b/utilities_common/util_base.py index ff5570735c..9bea158b59 100644 --- a/utilities_common/util_base.py +++ b/utilities_common/util_base.py @@ -24,6 +24,7 @@ def iter_namespace(ns_pkg): for _, module_name, ispkg in iter_namespace(plugins_namespace): if ispkg: + yield from self.load_plugins(importlib.import_module(module_name)) continue log.log_debug('importing plugin: {}'.format(module_name)) try: From 32e27a216e71428c5f6d2fa7a5c68314dea66825 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Mon, 17 May 2021 15:45:21 +0300 Subject: [PATCH 114/173] [sonic-cli-gen] fix show.py.j2 template Signed-off-by: Stepan Blyschak --- sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 b/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 index 58e7dfd2e0..ea82b9761f 100644 --- a/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 +++ b/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 @@ -77,7 +77,8 @@ def {{ name }}(db): body = [] table = db.cfgdb.get_table("{{ table.name }}") - for key, entry in natsort.natsorted(table).items(): + for key in natsort.natsorted(table): + entry = table[key] if not isinstance(key, tuple): key = (key,) From 02e6d27a4d56903ff8c91f27ce05d907c3ac694e Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Mon, 17 May 2021 13:29:20 +0300 Subject: [PATCH 115/173] [sonic-cli-gen] add remove plugin method Signed-off-by: Stepan Blyschak --- sonic_cli_gen/generator.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/sonic_cli_gen/generator.py b/sonic_cli_gen/generator.py index f64395c18b..adb6ac5a96 100644 --- a/sonic_cli_gen/generator.py +++ b/sonic_cli_gen/generator.py @@ -27,6 +27,12 @@ def generate_cli_plugin(self, cli_group, plugin_name): template = self.env.get_template(cli_group + '.py.j2') with open(plugin_path, 'w') as plugin_py: plugin_py.write(template.render(yang_dict)) + + def remove_cli_plugin(self, cli_group, plugin_name): + plugin_path = get_cli_plugin_path(cli_group, plugin_name + '_yang.py') + if os.path.exists(plugin_path): + os.remove(plugin_path) + def get_cli_plugin_path(command, plugin_name): pkg_loader = pkgutil.get_loader(f'{command}.plugins.auto') From ea97d666f9e1bb688e9e50547845480b5bf5ac0e Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Mon, 17 May 2021 16:51:51 +0300 Subject: [PATCH 116/173] python module name in plugins can be just simply a package name Signed-off-by: Stepan Blyschak --- sonic_package_manager/manager.py | 23 ++--------------------- 1 file changed, 2 insertions(+), 21 deletions(-) diff --git a/sonic_package_manager/manager.py b/sonic_package_manager/manager.py index 53884d2560..9eed20321e 100644 --- a/sonic_package_manager/manager.py +++ b/sonic_package_manager/manager.py @@ -2,7 +2,6 @@ import contextlib import functools -import hashlib import os import pkgutil import tempfile @@ -144,22 +143,6 @@ def parse_reference_expression(expression) -> PackageReference: return PackageReference.parse(expression) -def make_python_identifier(package: Package) -> str: - """ Generate unique python identifier from package name. - E.g: "sonic-package" and "sonic_package" are both valid package names, - while having single pythonized name "sonic_package". Hence, this function - calculates sha1 of package name and appends to the pythonized name. - - Args: - package: Package to generate python identifier for. - Returns: - Valid python identifier, unique for every package. - """ - - pythonized = utils.make_python_identifier(package.name) - return pythonized + hashlib.sha1(package.name.encode()).hexdigest() - - def get_cli_plugin_directory(command: str) -> str: """ Returns a plugins package directory for command group. @@ -176,19 +159,17 @@ def get_cli_plugin_directory(command: str) -> str: return plugins_pkg_path -def get_cli_plugin_path(package: Package, command: str, suffix: str = '') -> str: +def get_cli_plugin_path(package: Package, command: str) -> str: """ Returns a path where to put CLI plugin code. Args: package: Package to generate this path for. command: SONiC command: "show"/"config"/"clear". - suffix: Optional suffix for python plugin name. Returns: Path generated for this package. """ - plugin_module_name = make_python_identifier(package) + '_' + suffix - plugin_module_file = plugin_module_name + '.py' + plugin_module_file = package.name + '.py' return os.path.join(get_cli_plugin_directory(command), plugin_module_file) From cb7fa0696034f7c550ba33fff6681db876f8c376 Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Mon, 17 May 2021 15:00:12 +0000 Subject: [PATCH 117/173] refactored past test cases, added new Signed-off-by: Vadym Hlushko --- sonic_cli_gen/yang_parser.py | 3 +- .../cli_autogen_input/assert_dictionaries.py | 219 +++++++++++++++++- tests/cli_autogen_input/sonic-1-list.yang | 26 +++ .../sonic-1-object-container.yang | 20 ++ .../sonic-1-table-container.yang | 15 ++ tests/cli_autogen_input/sonic-2-lists.yang | 37 +++ .../sonic-2-object-containers.yang | 25 ++ .../sonic-2-table-containers.yang | 20 ++ .../sonic-many-table-containers.yang | 20 -- .../sonic-one-table-container.yang | 15 -- .../sonic-static-object-complex-1.yang | 49 ++++ .../sonic-static-object-complex-2.yang | 71 ++++++ .../cli_autogen_input/sonic-test-complex.yang | 115 +++++++++ tests/cli_autogen_input/sonic-vlan.yang | 190 --------------- tests/cli_autogen_yang_parser_test.py | 51 +++- 15 files changed, 630 insertions(+), 246 deletions(-) create mode 100644 tests/cli_autogen_input/sonic-1-list.yang create mode 100644 tests/cli_autogen_input/sonic-1-object-container.yang create mode 100644 tests/cli_autogen_input/sonic-1-table-container.yang create mode 100644 tests/cli_autogen_input/sonic-2-lists.yang create mode 100644 tests/cli_autogen_input/sonic-2-object-containers.yang create mode 100644 tests/cli_autogen_input/sonic-2-table-containers.yang delete mode 100644 tests/cli_autogen_input/sonic-many-table-containers.yang delete mode 100644 tests/cli_autogen_input/sonic-one-table-container.yang create mode 100644 tests/cli_autogen_input/sonic-static-object-complex-1.yang create mode 100644 tests/cli_autogen_input/sonic-static-object-complex-2.yang create mode 100644 tests/cli_autogen_input/sonic-test-complex.yang delete mode 100644 tests/cli_autogen_input/sonic-vlan.yang diff --git a/sonic_cli_gen/yang_parser.py b/sonic_cli_gen/yang_parser.py index 32e2b1a7f0..c77250f066 100644 --- a/sonic_cli_gen/yang_parser.py +++ b/sonic_cli_gen/yang_parser.py @@ -26,7 +26,7 @@ class YangParser: 'attrs': [ { 'name': 'value', - 'descruption': 'value', + 'description': 'value', 'is-leaf-list': False, 'is-mandatory': False } @@ -44,6 +44,7 @@ class YangParser: } In case if YANG model does NOT have a 'list' entity, it has the same structure as above, but 'dynamic-objects' changed to 'static-objects' and have no 'keys' """ + def __init__(self, yang_model_name, config_db_path, diff --git a/tests/cli_autogen_input/assert_dictionaries.py b/tests/cli_autogen_input/assert_dictionaries.py index 623233e7ee..9a2989d500 100644 --- a/tests/cli_autogen_input/assert_dictionaries.py +++ b/tests/cli_autogen_input/assert_dictionaries.py @@ -5,22 +5,21 @@ one_table_container = { "tables":[ { - "description":"FIRST_TABLE description", - "name":"FIRST_TABLE", + "description":"TABLE_1 description", + "name":"TABLE_1", "static-objects":[ { - } ] } ] } -many_table_containers = { +two_table_containers = { "tables":[ { - "description":"FIRST_TABLE description", - "name":"FIRST_TABLE", + "description":"TABLE_1 description", + "name":"TABLE_1", "static-objects":[ { @@ -28,8 +27,8 @@ ] }, { - "description":"SECOND_TABLE description", - "name":"SECOND_TABLE", + "description":"TABLE_2 description", + "name":"TABLE_2", "static-objects":[ { @@ -37,4 +36,208 @@ ] } ] +} + +one_object_container = { + "tables":[ + { + "description":"TABLE_1 description", + "name":"TABLE_1", + "static-objects":[ + { + "name":"OBJECT_1", + "description":"OBJECT_1 description", + "attrs":[ + ] + } + ] + } + ] +} + +two_object_containers = { + "tables":[ + { + "description":"FIRST_TABLE description", + "name":"TABLE_1", + "static-objects":[ + { + "name":"OBJECT_1", + "description":"OBJECT_1 description", + "attrs":[ + ] + }, + { + "name":"OBJECT_2", + "description":"OBJECT_2 description", + "attrs":[ + ] + } + ] + } + ] +} + +one_list = { + "tables":[ + { + "description":"TABLE_1 description", + "name":"TABLE_1", + "dynamic-objects":[ + { + "name":"TABLE_1_LIST", + "description":"TABLE_1_LIST description", + "keys":[ + { + "name": "key_name", + "description": "", + } + ], + "attrs":[ + ] + } + ] + } + ] +} + +two_lists = { + "tables":[ + { + "description":"TABLE_1 description", + "name":"TABLE_1", + "dynamic-objects":[ + { + "name":"TABLE_1_LIST_1", + "description":"TABLE_1_LIST_1 description", + "keys":[ + { + "name": "key_name1", + "description": "", + } + ], + "attrs":[ + ] + }, + { + "name":"TABLE_1_LIST_2", + "description":"TABLE_1_LIST_2 description", + "keys":[ + { + "name": "key_name2", + "description": "", + } + ], + "attrs":[ + ] + } + ] + } + ] +} + +static_object_complex_1 = { + "tables":[ + { + "description":"TABLE_1 description", + "name":"TABLE_1", + "static-objects":[ + { + "name":"OBJECT_1", + "description":"OBJECT_1 description", + "attrs":[ + { + "name":"OBJ_1_LEAF_1", + "description": "OBJ_1_LEAF_1 description", + "is-mandatory": False, + "is-leaf-list": False, + }, + { + "name":"OBJ_1_LEAF_LIST_1", + "description": "", + "is-mandatory": False, + "is-leaf-list": True, + }, + { + "name":"OBJ_1_CHOICE_1_LEAF_1", + "description": "", + "is-mandatory": False, + "is-leaf-list": False, + }, + { + "name":"OBJ_1_CHOICE_1_LEAF_2", + "description": "", + "is-mandatory": False, + "is-leaf-list": False, + } + ] + } + ] + } + ] +} + +static_object_complex_2 = { + "tables":[ + { + "description":"TABLE_1 description", + "name":"TABLE_1", + "static-objects":[ + { + "name":"OBJECT_1", + "description":"OBJECT_1 description", + "attrs":[ + { + "name":"OBJ_1_LEAF_1", + "description": "OBJ_1_LEAF_1 description", + "is-mandatory": False, + "is-leaf-list": False, + }, + { + "name":"OBJ_1_LEAF_2", + "description": "OBJ_1_LEAF_2 description", + "is-mandatory": False, + "is-leaf-list": False, + }, + { + "name":"OBJ_1_LEAF_LIST_1", + "description": "", + "is-mandatory": False, + "is-leaf-list": True, + }, + { + "name":"OBJ_1_LEAF_LIST_2", + "description": "", + "is-mandatory": False, + "is-leaf-list": True, + }, + { + "name":"OBJ_1_CHOICE_1_LEAF_1", + "description": "", + "is-mandatory": False, + "is-leaf-list": False, + }, + { + "name":"OBJ_1_CHOICE_1_LEAF_2", + "description": "", + "is-mandatory": False, + "is-leaf-list": False, + }, + { + "name":"OBJ_1_CHOICE_2_LEAF_1", + "description": "", + "is-mandatory": False, + "is-leaf-list": False, + }, + { + "name":"OBJ_1_CHOICE_2_LEAF_2", + "description": "", + "is-mandatory": False, + "is-leaf-list": False, + }, + ] + } + ] + } + ] } \ No newline at end of file diff --git a/tests/cli_autogen_input/sonic-1-list.yang b/tests/cli_autogen_input/sonic-1-list.yang new file mode 100644 index 0000000000..c7fc4ee824 --- /dev/null +++ b/tests/cli_autogen_input/sonic-1-list.yang @@ -0,0 +1,26 @@ +module sonic-1-list { + + yang-version 1.1; + + namespace "http://github.com/Azure/s-1-list"; + prefix s-1-list; + + container sonic-1-list { + + container TABLE_1 { + + description "TABLE_1 description"; + + list TABLE_1_LIST { + + description "TABLE_1_LIST description"; + + key "key_name"; + + leaf key_name { + type string; + } + } + } + } +} \ No newline at end of file diff --git a/tests/cli_autogen_input/sonic-1-object-container.yang b/tests/cli_autogen_input/sonic-1-object-container.yang new file mode 100644 index 0000000000..d52b2a8caf --- /dev/null +++ b/tests/cli_autogen_input/sonic-1-object-container.yang @@ -0,0 +1,20 @@ +module sonic-1-object-container { + + yang-version 1.1; + + namespace "http://github.com/Azure/s-1-object"; + prefix s-1-object; + + container sonic-1-object-container { + + container TABLE_1 { + + description "TABLE_1 description"; + + container OBJECT_1 { + + description "OBJECT_1 description"; + } + } + } +} \ No newline at end of file diff --git a/tests/cli_autogen_input/sonic-1-table-container.yang b/tests/cli_autogen_input/sonic-1-table-container.yang new file mode 100644 index 0000000000..8963148158 --- /dev/null +++ b/tests/cli_autogen_input/sonic-1-table-container.yang @@ -0,0 +1,15 @@ +module sonic-1-table-container { + + yang-version 1.1; + + namespace "http://github.com/Azure/s-1-table"; + prefix s-1-table; + + container sonic-1-table-container { + + container TABLE_1 { + + description "TABLE_1 description"; + } + } +} \ No newline at end of file diff --git a/tests/cli_autogen_input/sonic-2-lists.yang b/tests/cli_autogen_input/sonic-2-lists.yang new file mode 100644 index 0000000000..2a4cd42fd9 --- /dev/null +++ b/tests/cli_autogen_input/sonic-2-lists.yang @@ -0,0 +1,37 @@ +module sonic-2-lists { + + yang-version 1.1; + + namespace "http://github.com/Azure/s-2-lists"; + prefix s-2-lists; + + container sonic-2-lists { + + container TABLE_1 { + + description "TABLE_1 description"; + + list TABLE_1_LIST_1 { + + description "TABLE_1_LIST_1 description"; + + key "key_name1"; + + leaf key_name1 { + type string; + } + } + + list TABLE_1_LIST_2 { + + description "TABLE_1_LIST_2 description"; + + key "key_name2"; + + leaf key_name2 { + type string; + } + } + } + } +} \ No newline at end of file diff --git a/tests/cli_autogen_input/sonic-2-object-containers.yang b/tests/cli_autogen_input/sonic-2-object-containers.yang new file mode 100644 index 0000000000..1aaaeb1a19 --- /dev/null +++ b/tests/cli_autogen_input/sonic-2-object-containers.yang @@ -0,0 +1,25 @@ +module sonic-2-object-containers { + + yang-version 1.1; + + namespace "http://github.com/Azure/s-2-object"; + prefix s-2-object; + + container sonic-2-object-containers { + + container TABLE_1 { + + description "FIRST_TABLE description"; + + container OBJECT_1 { + + description "OBJECT_1 description"; + } + + container OBJECT_2 { + + description "OBJECT_2 description"; + } + } + } +} \ No newline at end of file diff --git a/tests/cli_autogen_input/sonic-2-table-containers.yang b/tests/cli_autogen_input/sonic-2-table-containers.yang new file mode 100644 index 0000000000..a3f13474b5 --- /dev/null +++ b/tests/cli_autogen_input/sonic-2-table-containers.yang @@ -0,0 +1,20 @@ +module sonic-2-table-containers { + + yang-version 1.1; + + namespace "http://github.com/Azure/s-2-table"; + prefix s-2-table; + + container sonic-2-table-containers { + + container TABLE_1 { + + description "TABLE_1 description"; + } + + container TABLE_2 { + + description "TABLE_2 description"; + } + } +} diff --git a/tests/cli_autogen_input/sonic-many-table-containers.yang b/tests/cli_autogen_input/sonic-many-table-containers.yang deleted file mode 100644 index 41a7d3c288..0000000000 --- a/tests/cli_autogen_input/sonic-many-table-containers.yang +++ /dev/null @@ -1,20 +0,0 @@ -module sonic-many-table-containers { - - yang-version 1.1; - - namespace "http://github.com/Azure/sonic-many-table-cont"; - prefix many-table-cont; - - container sonic-many-table-containers { - - container FIRST_TABLE { - - description "FIRST_TABLE description"; - } - - container SECOND_TABLE { - - description "SECOND_TABLE description"; - } - } -} diff --git a/tests/cli_autogen_input/sonic-one-table-container.yang b/tests/cli_autogen_input/sonic-one-table-container.yang deleted file mode 100644 index b04afa7ed9..0000000000 --- a/tests/cli_autogen_input/sonic-one-table-container.yang +++ /dev/null @@ -1,15 +0,0 @@ -module sonic-one-table-container { - - yang-version 1.1; - - namespace "http://github.com/Azure/sonic-one-table"; - prefix one-table; - - container sonic-one-table-container { - - container FIRST_TABLE { - - description "FIRST_TABLE description"; - } - } -} \ No newline at end of file diff --git a/tests/cli_autogen_input/sonic-static-object-complex-1.yang b/tests/cli_autogen_input/sonic-static-object-complex-1.yang new file mode 100644 index 0000000000..a7dfee86ab --- /dev/null +++ b/tests/cli_autogen_input/sonic-static-object-complex-1.yang @@ -0,0 +1,49 @@ +module sonic-static-object-complex-1 { + + yang-version 1.1; + + namespace "http://github.com/Azure/static-complex-1"; + prefix static-complex-1; + + container sonic-static-object-complex-1 { + /* sonic-static-object-complex-1 - top level container */ + + container TABLE_1 { + /* TABLE_1 - table container */ + + description "TABLE_1 description"; + + container OBJECT_1 { + /* OBJECT_1 - object container, it have: + * 1 leaf, + * 1 leaf-list + * 1 choice + */ + + description "OBJECT_1 description"; + + leaf OBJ_1_LEAF_1 { + description "OBJ_1_LEAF_1 description"; + type string; + } + + leaf-list OBJ_1_LEAF_LIST_1 { + type string; + } + + choice OBJ_1_CHOICE_1 { + case OBJ_1_CHOICE_1_CASE_1 { + leaf OBJ_1_CHOICE_1_LEAF_1 { + type uint16; + } + } + case OBJ_1_CHOICE_1_CASE_2 { + leaf OBJ_1_CHOICE_1_LEAF_2 { + type string; + } + } + } + } + } + } +} \ No newline at end of file diff --git a/tests/cli_autogen_input/sonic-static-object-complex-2.yang b/tests/cli_autogen_input/sonic-static-object-complex-2.yang new file mode 100644 index 0000000000..451a445ce6 --- /dev/null +++ b/tests/cli_autogen_input/sonic-static-object-complex-2.yang @@ -0,0 +1,71 @@ +module sonic-static-object-complex-2 { + + yang-version 1.1; + + namespace "http://github.com/Azure/static-complex-2"; + prefix static-complex-2; + + container sonic-static-object-complex-2 { + /* sonic-static-object-complex-2 - top level container */ + + container TABLE_1 { + /* TABLE_1 - table container */ + + description "TABLE_1 description"; + + container OBJECT_1 { + /* OBJECT_1 - object container, it have: + * 2 leafs, + * 2 leaf-lists, + * 2 choices + */ + + description "OBJECT_1 description"; + + leaf OBJ_1_LEAF_1 { + description "OBJ_1_LEAF_1 description"; + type string; + } + + leaf OBJ_1_LEAF_2 { + description "OBJ_1_LEAF_2 description"; + type string; + } + + leaf-list OBJ_1_LEAF_LIST_1 { + type string; + } + + leaf-list OBJ_1_LEAF_LIST_2 { + type string; + } + + choice OBJ_1_CHOICE_1 { + case OBJ_1_CHOICE_1_CASE_1 { + leaf OBJ_1_CHOICE_1_LEAF_1 { + type uint16; + } + } + case OBJ_1_CHOICE_1_CASE_2 { + leaf OBJ_1_CHOICE_1_LEAF_2 { + type string; + } + } + } + + choice OBJ_1_CHOICE_2 { + case OBJ_1_CHOICE_2_CASE_1 { + leaf OBJ_1_CHOICE_2_LEAF_1 { + type uint16; + } + } + case OBJ_1_CHOICE_2_CASE_2 { + leaf OBJ_1_CHOICE_2_LEAF_2 { + type string; + } + } + } + } + } + } +} \ No newline at end of file diff --git a/tests/cli_autogen_input/sonic-test-complex.yang b/tests/cli_autogen_input/sonic-test-complex.yang new file mode 100644 index 0000000000..e4485c9697 --- /dev/null +++ b/tests/cli_autogen_input/sonic-test-complex.yang @@ -0,0 +1,115 @@ +module sonic-test-1 { + + yang-version 1.1; + + namespace "http://github.com/Azure/sonic-test-1"; + prefix many-test-1; + + container sonic-test-1 { + /* sonic-test-1 - top level container, it have: + * 2 table containers. Table container - represent Config DB table name. + */ + + container TABLE_1 { + /* TABLE_1 - table container, it have: + * 2 object containers, Object container - represent Confg DB object name. + */ + description "TABLE_1 description"; + + container OBJECT_1 { + /* OBJECT_1 - object container, it have: + * 1 leaf, + * 1 leaf-list + * 1 choice + */ + description "OBJECT_1 description"; + + leaf OBJ_1_LEAF_1 { + description "OBJ_1_LEAF_1 description"; + type string; + } + + leaf-list OBJ_1_LEAF_LIST_1 { + mandatory true; + type string; + } + + choice OBJ_1_CH_1 { + case OBJ_1_CH_1_CASE_1 { + leaf OBJ_1_CH_1_LEAF_1 { + type uint16; + } + } + case OBJ_1_CH_1_CASE_2 { + leaf OBJ_1_CH_1_LEAF_2 { + type string; + } + } + } + } + + container OBJECT_2 { + /* OBJECT_2 - table container, it have: + * 2 leaf, + * 2 leaf-list + * 2 choice + */ + description "OBJECT_2 description"; + + leaf OBJ_2_LEAF_1 { + description "OBJ_2_LEAF_1 description"; + type string; + } + + leaf OBJ_2_LEAF_2 { + mandatory true; + type string; + } + + leaf-list OBJ_2_LEAF_LIST_1 { + description "OBJ_2_LEAF_LIST_1 description"; + mandatory true; + type string; + } + + leaf-list OBJ_2_LEAF_LIST_2 { + type string; + } + + choice OBJ_2_CH_1 { + case OBJ_2_CH_1_CASE_1 { + leaf OBJ_2_CH_1_LEAF_1 { + type uint16; + } + } + case OBJ_2_CH_1_CASE_2 { + leaf OBJ_2_CH_1_LEAF_2 { + type string { + type string; + } + } + } + } + + choice OBJ_2_CH_2 { + case OBJ_2_CH_2_CASE_1 { + leaf OBJ_2_CH_2_LEAF_1 { + type uint16; + } + } + case OBJ_2_CH_2_CASE_2 { + leaf OBJ_2_CH_2_LEAF_2 { + type string { + type string; + } + } + } + } + } + } + + container TABLE_2 { + + } + } +} \ No newline at end of file diff --git a/tests/cli_autogen_input/sonic-vlan.yang b/tests/cli_autogen_input/sonic-vlan.yang deleted file mode 100644 index 2962161ef0..0000000000 --- a/tests/cli_autogen_input/sonic-vlan.yang +++ /dev/null @@ -1,190 +0,0 @@ -module sonic-vlan { - - yang-version 1.1; - - namespace "http://github.com/Azure/sonic-vlan"; - prefix vlan; - - import ietf-inet-types { - prefix inet; - } - - import sonic-types { - prefix stypes; - revision-date 2019-07-01; - } - - import sonic-extension { - prefix ext; - revision-date 2019-07-01; - } - - import sonic-port { - prefix port; - revision-date 2019-07-01; - } - - import sonic-vrf { - prefix vrf; - } - - description "VLAN yang Module for SONiC OS"; - - revision 2021-03-30 { - description "Modify the type of vrf name"; - } - - revision 2019-07-01 { - description "First Revision"; - } - - container sonic-vlan { - - container VLAN_INTERFACE { - - description "VLAN_INTERFACE part of config_db.json"; - - list VLAN_INTERFACE_LIST { - - description "VLAN INTERFACE part of config_db.json with vrf"; - - key "name"; - - leaf name { - type leafref { - path /vlan:sonic-vlan/vlan:VLAN/vlan:VLAN_LIST/vlan:name; - } - } - - leaf vrf_name { - type leafref{ - path "/vrf:sonic-vrf/vrf:VRF/vrf:VRF_LIST/vrf:name"; - } - } - } - /* end of VLAN_INTERFACE_LIST */ - - list VLAN_INTERFACE_IPPREFIX_LIST { - - key "name ip-prefix"; - - leaf name { - /* This node must be present in VLAN_INTERFACE_LIST */ - must "(current() = ../../VLAN_INTERFACE_LIST[name=current()]/name)" - { - error-message "Must condition not satisfied, Try adding Vlan: {}, Example: 'Vlan100': {}"; - } - - type leafref { - path "/vlan:sonic-vlan/vlan:VLAN/vlan:VLAN_LIST/vlan:name"; - } - } - - leaf ip-prefix { - type union { - type stypes:sonic-ip4-prefix; - type stypes:sonic-ip6-prefix; - } - } - - leaf scope { - type enumeration { - enum global; - enum local; - } - } - - leaf family { - - /* family leaf needed for backward compatibility - Both ip4 and ip6 address are string in IETF RFC 6021, - so must statement can check based on : or ., family - should be IPv4 or IPv6 according. - */ - - must "(contains(../ip-prefix, ':') and current()='IPv6') or - (contains(../ip-prefix, '.') and current()='IPv4')"; - type stypes:ip-family; - } - } - /* end of VLAN_INTERFACE_LIST */ - } - /* end of VLAN_INTERFACE container */ - - container VLAN { - - description "VLAN part of config_db.json"; - - list VLAN_LIST { - - key "name"; - - leaf name { - type string { - pattern 'Vlan([0-9]{1,3}|[1-3][0-9]{3}|[4][0][0-8][0-9]|[4][0][9][0-4])'; - } - } - - leaf vlanid { - type uint16 { - range 1..4094; - } - } - - leaf description { - type string { - length 1..255; - } - } - - leaf-list dhcp_servers { - type inet:ip-address; - } - - leaf mtu { - type uint16 { - range 1..9216; - } - } - - leaf admin_status { - type stypes:admin_status; - } - } - /* end of VLAN_LIST */ - } - /* end of container VLAN */ - - container VLAN_MEMBER { - - description "VLAN_MEMBER part of config_db.json"; - - list VLAN_MEMBER_LIST { - - key "name port"; - - leaf name { - type leafref { - path "/vlan:sonic-vlan/vlan:VLAN/vlan:VLAN_LIST/vlan:name"; - } - } - - leaf port { - /* key elements are mandatory by default */ - type leafref { - path /port:sonic-port/port:PORT/port:PORT_LIST/port:name; - } - } - - leaf tagging_mode { - mandatory true; - type stypes:vlan_tagging_mode; - } - } - /* end of list VLAN_MEMBER_LIST */ - } - /* end of container VLAN_MEMBER */ - } - /* end of container sonic-vlan */ -} -/* end of module sonic-vlan */ diff --git a/tests/cli_autogen_yang_parser_test.py b/tests/cli_autogen_yang_parser_test.py index e9010c4877..76375f0fa3 100644 --- a/tests/cli_autogen_yang_parser_test.py +++ b/tests/cli_autogen_yang_parser_test.py @@ -17,14 +17,41 @@ class TestYangParser: - - def test_one_table_container(self): - yang_model_name = 'sonic-one-table-container' - template('sonic-one-table-container', assert_dictionaries.one_table_container) - - def test_many_table_containers(self): - yang_model_name = 'sonic-many-table-containers' - template('sonic-many-table-containers', assert_dictionaries.many_table_containers) + #def test_1_table_container(self): + # yang_model_name = 'sonic-1-table-container' + # template(yang_model_name, assert_dictionaries.one_table_container) + # + #def test_2_table_containers(self): + # yang_model_name = 'sonic-2-table-containers' + # template(yang_model_name, assert_dictionaries.two_table_containers) + + #def test_1_object_container(self): + # yang_model_name = 'sonic-1-object-container' + # template(yang_model_name, assert_dictionaries.one_object_container) + + #def test_2_object_containers(self): + # yang_model_name = 'sonic-2-object-containers' + # template(yang_model_name, assert_dictionaries.two_object_containers) + + #def test_1_list(self): + # yang_model_name = 'sonic-1-list' + # template(yang_model_name, assert_dictionaries.one_list) + + #def test_2_lists(self): + # yang_model_name = 'sonic-2-lists' + # template(yang_model_name, assert_dictionaries.two_lists) + + #def test_static_object_complex_1(self): + # """ Test object container with: 1 leaf, 1 leaf-list, 1 choice. + # """ + # yang_model_name = 'sonic-static-object-complex-1' + # template(yang_model_name, assert_dictionaries.static_object_complex_1) + + #def test_static_object_complex_2(self): + # """ Test object container with: 2 leafs, 2 leaf-lists, 2 choices. + # """ + # yang_model_name = 'sonic-static-object-complex-2' + # template(yang_model_name, assert_dictionaries.static_object_complex_2) def template(yang_model_name, correct_dict): config_db_path = os.path.join(test_path, 'cli_autogen_input/config_db.json') @@ -34,8 +61,8 @@ def template(yang_model_name, correct_dict): allow_tbl_without_yang = True, debug = False) yang_dict = parser.parse_yang_model() - # debug - pretty_log(yang_dict) + + pretty_log_debug(yang_dict) assert yang_dict == correct_dict @@ -62,7 +89,7 @@ def remove_yang_model(yang_model_name): os.system(cmd) # DEBUG function -def pretty_log(dictionary): +def pretty_log_debug(dictionary): for line in pprint.pformat(dictionary).split('\n'): - logging.warning(line) + logging.debug(line) From 5737f5f03b23547beca498b8cdd31f4380246a96 Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Mon, 17 May 2021 15:23:50 +0000 Subject: [PATCH 118/173] added test_dynamic_object_complex Signed-off-by: Vadym Hlushko --- .../cli_autogen_input/assert_dictionaries.py | 122 ++++++++++++++++++ .../sonic-dynamic-object-complex-1.yang | 57 ++++++++ .../sonic-dynamic-object-complex-2.yang | 84 ++++++++++++ tests/cli_autogen_yang_parser_test.py | 12 ++ 4 files changed, 275 insertions(+) create mode 100644 tests/cli_autogen_input/sonic-dynamic-object-complex-1.yang create mode 100644 tests/cli_autogen_input/sonic-dynamic-object-complex-2.yang diff --git a/tests/cli_autogen_input/assert_dictionaries.py b/tests/cli_autogen_input/assert_dictionaries.py index 9a2989d500..3aac3c2f6f 100644 --- a/tests/cli_autogen_input/assert_dictionaries.py +++ b/tests/cli_autogen_input/assert_dictionaries.py @@ -240,4 +240,126 @@ ] } ] +} + +dynamic_object_complex_1 = { + "tables":[ + { + "description":"TABLE_1 description", + "name":"TABLE_1", + "dynamic-objects":[ + { + "name":"OBJECT_1_LIST", + "description":"OBJECT_1_LIST description", + "attrs":[ + { + "name":"OBJ_1_LEAF_1", + "description": "OBJ_1_LEAF_1 description", + "is-mandatory": False, + "is-leaf-list": False, + }, + { + "name":"OBJ_1_LEAF_LIST_1", + "description": "", + "is-mandatory": False, + "is-leaf-list": True, + }, + { + "name":"OBJ_1_CHOICE_1_LEAF_1", + "description": "", + "is-mandatory": False, + "is-leaf-list": False, + }, + { + "name":"OBJ_1_CHOICE_1_LEAF_2", + "description": "", + "is-mandatory": False, + "is-leaf-list": False, + } + ], + "keys":[ + { + "name": "KEY_LEAF_1", + "description": "KEY_LEAF_1 description", + } + ] + } + ] + } + ] +} + +dynamic_object_complex_2 = { + "tables":[ + { + "description":"TABLE_1 description", + "name":"TABLE_1", + "dynamic-objects":[ + { + "name":"OBJECT_1_LIST", + "description":"OBJECT_1_LIST description", + "attrs":[ + { + "name":"OBJ_1_LEAF_1", + "description": "OBJ_1_LEAF_1 description", + "is-mandatory": False, + "is-leaf-list": False, + }, + { + "name":"OBJ_1_LEAF_2", + "description": "OBJ_1_LEAF_2 description", + "is-mandatory": False, + "is-leaf-list": False, + }, + { + "name":"OBJ_1_LEAF_LIST_1", + "description": "", + "is-mandatory": False, + "is-leaf-list": True, + }, + { + "name":"OBJ_1_LEAF_LIST_2", + "description": "", + "is-mandatory": False, + "is-leaf-list": True, + }, + { + "name":"OBJ_1_CHOICE_1_LEAF_1", + "description": "", + "is-mandatory": False, + "is-leaf-list": False, + }, + { + "name":"OBJ_1_CHOICE_1_LEAF_2", + "description": "", + "is-mandatory": False, + "is-leaf-list": False, + }, + { + "name":"OBJ_1_CHOICE_2_LEAF_1", + "description": "", + "is-mandatory": False, + "is-leaf-list": False, + }, + { + "name":"OBJ_1_CHOICE_2_LEAF_2", + "description": "", + "is-mandatory": False, + "is-leaf-list": False, + } + ], + "keys":[ + { + "name": "KEY_LEAF_1", + "description": "KEY_LEAF_1 description", + }, + { + "name": "KEY_LEAF_2", + "description": "KEY_LEAF_2 description", + } + ] + } + ] + } + ] } \ No newline at end of file diff --git a/tests/cli_autogen_input/sonic-dynamic-object-complex-1.yang b/tests/cli_autogen_input/sonic-dynamic-object-complex-1.yang new file mode 100644 index 0000000000..9beb98549d --- /dev/null +++ b/tests/cli_autogen_input/sonic-dynamic-object-complex-1.yang @@ -0,0 +1,57 @@ +module sonic-dynamic-object-complex-1 { + + yang-version 1.1; + + namespace "http://github.com/Azure/dynamic-complex-1"; + prefix dynamic-complex-1; + + container sonic-dynamic-object-complex-1 { + /* sonic-dynamic-object-complex-1 - top level container */ + + container TABLE_1 { + /* TABLE_1 - table container */ + + description "TABLE_1 description"; + + list OBJECT_1_LIST { + /* OBJECT_1_LIST - dynamic object container, it have: + * 1 key, + * 1 leaf, + * 1 leaf-list + * 1 choice + */ + + description "OBJECT_1_LIST description"; + + key "KEY_LEAF_1"; + + leaf KEY_LEAF_1 { + description "KEY_LEAF_1 description"; + type string; + } + + leaf OBJ_1_LEAF_1 { + description "OBJ_1_LEAF_1 description"; + type string; + } + + leaf-list OBJ_1_LEAF_LIST_1 { + type string; + } + + choice OBJ_1_CHOICE_1 { + case OBJ_1_CHOICE_1_CASE_1 { + leaf OBJ_1_CHOICE_1_LEAF_1 { + type uint16; + } + } + case OBJ_1_CHOICE_1_CASE_2 { + leaf OBJ_1_CHOICE_1_LEAF_2 { + type string; + } + } + } + } + } + } +} \ No newline at end of file diff --git a/tests/cli_autogen_input/sonic-dynamic-object-complex-2.yang b/tests/cli_autogen_input/sonic-dynamic-object-complex-2.yang new file mode 100644 index 0000000000..00e25c8135 --- /dev/null +++ b/tests/cli_autogen_input/sonic-dynamic-object-complex-2.yang @@ -0,0 +1,84 @@ +module sonic-dynamic-object-complex-2 { + + yang-version 1.1; + + namespace "http://github.com/Azure/dynamic-complex-2"; + prefix dynamic-complex-2; + + container sonic-dynamic-object-complex-2 { + /* sonic-dynamic-object-complex-2 - top level container */ + + container TABLE_1 { + /* TABLE_1 - table container */ + + description "TABLE_1 description"; + + list OBJECT_1_LIST { + /* OBJECT_1_LIST - dynamic object container, it have: + * 2 keys + * 2 leaf, + * 2 leaf-list + * 2 choice + */ + + description "OBJECT_1_LIST description"; + + key "KEY_LEAF_1 KEY_LEAF_2"; + + leaf KEY_LEAF_1 { + description "KEY_LEAF_1 description"; + type string; + } + + leaf KEY_LEAF_2 { + description "KEY_LEAF_2 description"; + type string; + } + + leaf OBJ_1_LEAF_1 { + description "OBJ_1_LEAF_1 description"; + type string; + } + + leaf OBJ_1_LEAF_2 { + description "OBJ_1_LEAF_2 description"; + type string; + } + + leaf-list OBJ_1_LEAF_LIST_1 { + type string; + } + + leaf-list OBJ_1_LEAF_LIST_2 { + type string; + } + + choice OBJ_1_CHOICE_1 { + case OBJ_1_CHOICE_1_CASE_1 { + leaf OBJ_1_CHOICE_1_LEAF_1 { + type uint16; + } + } + case OBJ_1_CHOICE_1_CASE_2 { + leaf OBJ_1_CHOICE_1_LEAF_2 { + type string; + } + } + } + + choice OBJ_1_CHOICE_2 { + case OBJ_1_CHOICE_2_CASE_1 { + leaf OBJ_1_CHOICE_2_LEAF_1 { + type uint16; + } + } + case OBJ_1_CHOICE_2_CASE_2 { + leaf OBJ_1_CHOICE_2_LEAF_2 { + type string; + } + } + } + } + } + } +} \ No newline at end of file diff --git a/tests/cli_autogen_yang_parser_test.py b/tests/cli_autogen_yang_parser_test.py index 76375f0fa3..acabfe5082 100644 --- a/tests/cli_autogen_yang_parser_test.py +++ b/tests/cli_autogen_yang_parser_test.py @@ -53,6 +53,18 @@ class TestYangParser: # yang_model_name = 'sonic-static-object-complex-2' # template(yang_model_name, assert_dictionaries.static_object_complex_2) + #def test_dynamic_object_complex_1(self): + # """ Test object container with: 1 key, 1 leaf, 1 leaf-list, 1 choice. + # """ + # yang_model_name = 'sonic-dynamic-object-complex-1' + # template(yang_model_name, assert_dictionaries.dynamic_object_complex_1) + + def test_dynamic_object_complex_2(self): + """ Test object container with: 2 keys, 2 leafs, 2 leaf-list, 2 choice. + """ + yang_model_name = 'sonic-dynamic-object-complex-2' + template(yang_model_name, assert_dictionaries.dynamic_object_complex_2) + def template(yang_model_name, correct_dict): config_db_path = os.path.join(test_path, 'cli_autogen_input/config_db.json') move_yang_model(yang_model_name) From a56bbb09375b88ab881915e3b569c4b4dfee78fc Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Tue, 18 May 2021 14:54:28 +0300 Subject: [PATCH 119/173] [sonic-cli-gen] implement grouping in show template Signed-off-by: Stepan Blyschak --- .../templates/sonic-cli-gen/show.py.j2 | 55 ++++++++++++++----- 1 file changed, 41 insertions(+), 14 deletions(-) diff --git a/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 b/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 index ea82b9761f..3859573d72 100644 --- a/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 +++ b/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 @@ -6,19 +6,46 @@ import tabulate import natsort import utilities_common.cli as clicommon -{% macro print_attr(attr) %} -{%- if not attr["is-leaf-list"] %} -entry.get("{{ attr.name }}", "N/A") -{%- else %} -"\n".join(entry.get("{{ attr.name }}", [])) -{%- endif %} -{% endmacro %} + +{% macro column_name(name) -%} +{{ name|upper|replace("_", " ")|replace("-", " ") }} +{%- endmacro %} + + +def print_attr_helper(entry, attr): + if attr["is-leaf-list"]: + return "\n".join(entry.get(attr["name"], [])) + return entry.get(attr["name"], "N/A") + + +def print_group_helper(entry, attrs): + data = [] + for attr in attrs: + if entry.get(attr["name"]): + data.append((attr["name"] + ":", print_attr_helper(entry, attr))) + return tabulate.tabulate(data, tablefmt="plain") -{% macro gen_header(attrs) %} -{% for attr in attrs %} -"{{ attr.name|upper|replace("_", " ")|replace("-", " ") }}", +{% macro gen_row(entry, attrs) -%} +[ +{%- for attr in attrs|rejectattr("group", "defined") %} +print_attr_helper({{ entry }}, {{ attr }}), +{%- endfor %} +{%- for group, attrs in attrs|selectattr("group", "defined")|groupby("group") %} +print_group_helper({{ entry }}, {{ attrs }}), +{%- endfor %} +] +{% endmacro %} + +{% macro gen_header(attrs) -%} +[ +{% for attr in attrs|rejectattr("group", "defined") %} +"{{ column_name(attr.name) }}", +{% endfor %} +{% for group, attrs in attrs|selectattr("group", "defined")|groupby("group") %} +"{{ column_name(group) }}", {% endfor %} +] {% endmacro %} @@ -37,12 +64,12 @@ def {{ table.name }}(): def {{ table.name }}_{{ object.name }}(db): """ {{ object.description }} """ - header = [{{ gen_header(object.attrs) }}] + header = {{ gen_header(object.attrs) }} body = [] table = db.cfgdb.get_table("{{ table.name }}") entry = table.get("{{ object.name }}", {}) - row = [{%- for attr in object.attrs -%} {{ print_attr(attr) }}, {%- endfor %}] + row = {{ gen_row("entry", object.attrs) }} body.append(row) click.echo(tabulate.tabulate(body, header)) @@ -73,7 +100,7 @@ def {{ table.name }}(): def {{ name }}(db): """ {{ object.description }} """ - header = [{{ gen_header(object["keys"] + object.attrs) }}] + header = {{ gen_header(object["keys"] + object.attrs) }} body = [] table = db.cfgdb.get_table("{{ table.name }}") @@ -82,7 +109,7 @@ def {{ name }}(db): if not isinstance(key, tuple): key = (key,) - row = [*key, {%- for attr in object.attrs -%} {{ print_attr(attr) }}, {%- endfor %}] + row = [*key] + {{ gen_row("entry", object.attrs) }} body.append(row) click.echo(tabulate.tabulate(body, header)) From 7b167a6b5ad0c1a5e89b47c7cc0daada4e24f417 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Tue, 18 May 2021 14:54:28 +0300 Subject: [PATCH 120/173] [sonic-cli-gen] implement grouping in show template Signed-off-by: Stepan Blyschak --- .../templates/sonic-cli-gen/show.py.j2 | 55 ++++++++++++++----- 1 file changed, 41 insertions(+), 14 deletions(-) diff --git a/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 b/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 index ea82b9761f..3859573d72 100644 --- a/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 +++ b/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 @@ -6,19 +6,46 @@ import tabulate import natsort import utilities_common.cli as clicommon -{% macro print_attr(attr) %} -{%- if not attr["is-leaf-list"] %} -entry.get("{{ attr.name }}", "N/A") -{%- else %} -"\n".join(entry.get("{{ attr.name }}", [])) -{%- endif %} -{% endmacro %} + +{% macro column_name(name) -%} +{{ name|upper|replace("_", " ")|replace("-", " ") }} +{%- endmacro %} + + +def print_attr_helper(entry, attr): + if attr["is-leaf-list"]: + return "\n".join(entry.get(attr["name"], [])) + return entry.get(attr["name"], "N/A") + + +def print_group_helper(entry, attrs): + data = [] + for attr in attrs: + if entry.get(attr["name"]): + data.append((attr["name"] + ":", print_attr_helper(entry, attr))) + return tabulate.tabulate(data, tablefmt="plain") -{% macro gen_header(attrs) %} -{% for attr in attrs %} -"{{ attr.name|upper|replace("_", " ")|replace("-", " ") }}", +{% macro gen_row(entry, attrs) -%} +[ +{%- for attr in attrs|rejectattr("group", "defined") %} +print_attr_helper({{ entry }}, {{ attr }}), +{%- endfor %} +{%- for group, attrs in attrs|selectattr("group", "defined")|groupby("group") %} +print_group_helper({{ entry }}, {{ attrs }}), +{%- endfor %} +] +{% endmacro %} + +{% macro gen_header(attrs) -%} +[ +{% for attr in attrs|rejectattr("group", "defined") %} +"{{ column_name(attr.name) }}", +{% endfor %} +{% for group, attrs in attrs|selectattr("group", "defined")|groupby("group") %} +"{{ column_name(group) }}", {% endfor %} +] {% endmacro %} @@ -37,12 +64,12 @@ def {{ table.name }}(): def {{ table.name }}_{{ object.name }}(db): """ {{ object.description }} """ - header = [{{ gen_header(object.attrs) }}] + header = {{ gen_header(object.attrs) }} body = [] table = db.cfgdb.get_table("{{ table.name }}") entry = table.get("{{ object.name }}", {}) - row = [{%- for attr in object.attrs -%} {{ print_attr(attr) }}, {%- endfor %}] + row = {{ gen_row("entry", object.attrs) }} body.append(row) click.echo(tabulate.tabulate(body, header)) @@ -73,7 +100,7 @@ def {{ table.name }}(): def {{ name }}(db): """ {{ object.description }} """ - header = [{{ gen_header(object["keys"] + object.attrs) }}] + header = {{ gen_header(object["keys"] + object.attrs) }} body = [] table = db.cfgdb.get_table("{{ table.name }}") @@ -82,7 +109,7 @@ def {{ name }}(db): if not isinstance(key, tuple): key = (key,) - row = [*key, {%- for attr in object.attrs -%} {{ print_attr(attr) }}, {%- endfor %}] + row = [*key] + {{ gen_row("entry", object.attrs) }} body.append(row) click.echo(tabulate.tabulate(body, header)) From 16e0ccac525a0d5e0257fe71b509681cd7c24596 Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Tue, 18 May 2021 15:01:36 +0000 Subject: [PATCH 121/173] Reworked 'grouping' parser Signed-off-by: Vadym Hlushko --- sonic_cli_gen/generator.py | 9 +- sonic_cli_gen/yang_parser.py | 166 ++++++++++++++++++-------- tests/cli_autogen_yang_parser_test.py | 86 ++++++------- 3 files changed, 168 insertions(+), 93 deletions(-) diff --git a/sonic_cli_gen/generator.py b/sonic_cli_gen/generator.py index f3fcf0e62b..37924b1ce5 100644 --- a/sonic_cli_gen/generator.py +++ b/sonic_cli_gen/generator.py @@ -26,10 +26,11 @@ def generate_cli_plugin(self, cli_group, plugin_name): allow_tbl_without_yang=True, debug=False) yang_dict = parser.parse_yang_model() - plugin_path = get_cli_plugin_path(cli_group, plugin_name + '_yang.py') - template = self.env.get_template(cli_group + '.py.j2') - with open(plugin_path, 'w') as plugin_py: - plugin_py.write(template.render(yang_dict)) + import pprint; pprint.pprint(yang_dict) + #plugin_path = get_cli_plugin_path(cli_group, plugin_name + '_yang.py') + #template = self.env.get_template(cli_group + '.py.j2') + #with open(plugin_path, 'w') as plugin_py: + # plugin_py.write(template.render(yang_dict)) def get_cli_plugin_path(command, plugin_name): pkg_loader = pkgutil.get_loader(f'{command}.plugins') diff --git a/sonic_cli_gen/yang_parser.py b/sonic_cli_gen/yang_parser.py index c77250f066..bb05d5a21a 100644 --- a/sonic_cli_gen/yang_parser.py +++ b/sonic_cli_gen/yang_parser.py @@ -118,17 +118,17 @@ def parse_yang_model(self) -> dict: # 'table' container it is a container that goes after 'top level' container if isinstance(self.y_table_containers, list): for tbl_cont in self.y_table_containers: - y2d_elem = on_table_container(self.y_module, tbl_cont) + y2d_elem = on_table_container(self.y_module, tbl_cont, self.conf_mgmt) self.yang_2_dict['tables'].append(y2d_elem) else: - y2d_elem = on_table_container(self.y_module, self.y_table_containers) + y2d_elem = on_table_container(self.y_module, self.y_table_containers, self.conf_mgmt) self.yang_2_dict['tables'].append(y2d_elem) return self.yang_2_dict #------------------------------HANDLERS--------------------------------# -def on_table_container(y_module: OrderedDict, tbl_cont: OrderedDict) -> dict: +def on_table_container(y_module: OrderedDict, tbl_cont: OrderedDict, conf_mgmt) -> dict: """ Parse 'table' container, 'table' container goes after 'top level' container @@ -153,10 +153,10 @@ def on_table_container(y_module: OrderedDict, tbl_cont: OrderedDict) -> dict: obj_cont = tbl_cont.get('container') if isinstance(obj_cont, list): for cont in obj_cont: - static_obj_elem = on_object_container(y_module, cont, is_list=False) + static_obj_elem = on_object_container(y_module, cont, conf_mgmt, is_list=False) y2d_elem['static-objects'].append(static_obj_elem) else: - static_obj_elem = on_object_container(y_module, obj_cont, is_list=False) + static_obj_elem = on_object_container(y_module, obj_cont, conf_mgmt, is_list=False) y2d_elem['static-objects'].append(static_obj_elem) else: y2d_elem['dynamic-objects'] = list() @@ -164,10 +164,10 @@ def on_table_container(y_module: OrderedDict, tbl_cont: OrderedDict) -> dict: # 'container' can have more than 1 'list' entity if isinstance(tbl_cont_lists, list): for _list in tbl_cont_lists: - dynamic_obj_elem = on_object_container(y_module, _list, is_list=True) + dynamic_obj_elem = on_object_container(y_module, _list, conf_mgmt, is_list=True) y2d_elem['dynamic-objects'].append(dynamic_obj_elem) else: - dynamic_obj_elem = on_object_container(y_module, tbl_cont_lists, is_list=True) + dynamic_obj_elem = on_object_container(y_module, tbl_cont_lists, conf_mgmt, is_list=True) y2d_elem['dynamic-objects'].append(dynamic_obj_elem) # move 'keys' elements from 'attrs' to 'keys' @@ -175,7 +175,7 @@ def on_table_container(y_module: OrderedDict, tbl_cont: OrderedDict) -> dict: return y2d_elem -def on_object_container(y_module: OrderedDict, cont: OrderedDict, is_list: bool) -> dict: +def on_object_container(y_module: OrderedDict, cont: OrderedDict, conf_mgmt, is_list: bool) -> dict: """ Parse a 'object container'. 'Object container' represent OBJECT inside Config DB schema: { @@ -208,58 +208,47 @@ def on_object_container(y_module: OrderedDict, cont: OrderedDict, is_list: bool) attrs_list = list() attrs_list.extend(get_leafs(cont)) attrs_list.extend(get_leaf_lists(cont)) - attrs_list.extend(get_choices(y_module, cont)) + attrs_list.extend(get_choices(y_module, cont, conf_mgmt)) # TODO: need to test 'grouping' - #attrs_list.extend(get_uses_grouping(y_module, cont)) + attrs_list.extend(get_uses(y_module, cont, conf_mgmt)) obj_elem['attrs'] = attrs_list return obj_elem -def on_grouping(y_module: OrderedDict, y_grouping, y_uses) -> list: - """ Parse a YANG 'grouping' and 'uses' entities - 'grouping' element can have - 'leaf', 'leaf-list', 'choice' +def on_uses(y_module: OrderedDict, y_uses, conf_mgmt) -> list: + """ Parse a YANG 'uses' entities + 'uses' refearing to 'grouping' YANG entity Args: y_module: reference to 'module' - y_grouping: reference to 'grouping' y_uses: reference to 'uses' Returns: dictionary - element for obj_elem['attrs'], 'attrs' contain a parsed 'leafs' """ - ret_attrs = list() + y_grouping = get_all_grouping(y_module, y_uses, conf_mgmt) - if isinstance(y_uses, list): - if isinstance(y_grouping, list): - for use in y_uses: - for group in y_grouping: - if use.get('@name') == group.get('@name'): - ret_attrs.extend(get_leafs(group)) - ret_attrs.extend(get_leaf_lists(group)) - ret_attrs.extend(get_choices(y_module, group)) - else: + if y_grouping == []: + # not sure if it can happend + raise Exception('EMPTY') + + for group in y_grouping: + if isinstance(y_uses, list): for use in y_uses: - if use.get('@name') == y_grouping.get('@name'): - ret_attrs.extend(get_leafs(y_grouping)) - ret_attrs.extend(get_leaf_lists(y_grouping)) - ret_attrs.extend(get_choices(y_module, y_grouping)) - else: - if isinstance(y_grouping, list): - for group in y_grouping: - if y_uses.get('@name') == group.get('@name'): + if use.get('@name') == group.get('@name'): ret_attrs.extend(get_leafs(group)) ret_attrs.extend(get_leaf_lists(group)) - ret_attrs.extend(get_choices(y_module, group)) + ret_attrs.extend(get_choices(y_module, group, conf_mgmt)) else: - if y_uses.get('@name') == y_grouping.get('@name'): - ret_attrs.extend(get_leafs(y_grouping)) - ret_attrs.extend(get_leaf_lists(y_grouping)) - ret_attrs.extend(get_choices(y_module, y_grouping)) + if y_uses.get('@name') == group.get('@name'): + ret_attrs.extend(get_leafs(group)) + ret_attrs.extend(get_leaf_lists(group)) + ret_attrs.extend(get_choices(y_module, group, conf_mgmt)) return ret_attrs -def on_choices(y_module: OrderedDict, y_choices) -> list: +def on_choices(y_module: OrderedDict, y_choices, conf_mgmt) -> list: """ Parse a YANG 'choice' entities Args: @@ -270,17 +259,17 @@ def on_choices(y_module: OrderedDict, y_choices) -> list: ret_attrs = list() - # the YANG model can have multiple 'choice' entities inside 'container' or 'list' + # the YANG model can have multiple 'choice' entities inside a 'container' or 'list' if isinstance(y_choices, list): for choice in y_choices: - attrs = on_choice_cases(y_module, choice.get('case')) + attrs = on_choice_cases(y_module, choice.get('case'), conf_mgmt) ret_attrs.extend(attrs) else: - ret_attrs = on_choice_cases(y_module, y_choices.get('case')) + ret_attrs = on_choice_cases(y_module, y_choices.get('case'), conf_mgmt) return ret_attrs -def on_choice_cases(y_module: OrderedDict, y_cases: list) -> list: +def on_choice_cases(y_module: OrderedDict, y_cases: list, conf_mgmt) -> list: """ Parse a single YANG 'case' entity from 'choice' entity 'case' element can have inside - 'leaf', 'leaf-list', 'uses' @@ -298,7 +287,7 @@ def on_choice_cases(y_module: OrderedDict, y_cases: list) -> list: ret_attrs.extend(get_leafs(case)) ret_attrs.extend(get_leaf_lists(case)) # TODO: need to deeply test it - #ret_attrs.extend(get_uses_grouping(y_module, case)) + ret_attrs.extend(get_uses(y_module, case, conf_mgmt)) else: raise Exception('It has no sense to using a single "case" element inside "choice" element') @@ -375,18 +364,99 @@ def get_leaf_lists(y_entity: OrderedDict) -> list: return [] -def get_choices(y_module: OrderedDict, y_entity: OrderedDict) -> list: +def get_choices(y_module: OrderedDict, y_entity: OrderedDict, conf_mgmt) -> list: if y_entity.get('choice') is not None: - return on_choices(y_module, y_entity.get('choice')) + return on_choices(y_module, y_entity.get('choice'), conf_mgmt) return [] -def get_uses_grouping(y_module: OrderedDict, y_entity: OrderedDict) -> list: - if y_entity.get('uses') is not None and y_module.get('grouping') is not None: - return on_grouping(y_module, y_module.get('grouping'), y_entity.get('uses')) +def get_uses(y_module: OrderedDict, y_entity: OrderedDict, conf_mgmt) -> list: + if y_entity.get('uses') is not None: + return on_uses(y_module, y_entity.get('uses'), conf_mgmt) return [] +def get_all_grouping(y_module: OrderedDict, y_uses: OrderedDict, conf_mgmt) -> list: + # WARNING + # TODO add to the design statement that grouping should be defined under the 'module' and NOT in nested containers + ret_grouping = list() + prefix_list = get_import_prefixes(y_uses) + + # in case if 'grouping' located in the same YANG model + local_grouping = y_module.get('grouping') + if local_grouping is not None: + if isinstance(local_grouping, list): + for group in local_grouping: + ret_grouping.append(group) + else: + ret_grouping.append(local_grouping) + + # if prefix_list is NOT empty it means that 'grouping' was imported from another YANG model + if prefix_list != []: + for prefix in prefix_list: + y_import = y_module.get('import') + if isinstance(y_import, list): + for _import in y_import: + if _import.get('prefix').get('@value') == prefix: + ret_grouping.extend(get_grouping_from_another_yang_model(_import.get('@module'), conf_mgmt)) + else: + if y_import.get('prefix').get('@value') == prefix: + ret_grouping.extend(get_grouping_from_another_yang_model(y_import.get('@module'), conf_mgmt)) + + return ret_grouping + +def get_grouping_from_another_yang_model(yang_model_name: str, conf_mgmt) -> list: + """ Get the YANG 'grouping' entity + + Args: + yang_model_name - YANG model to search + conf_mgmt - reference to ConfigMgmt class instance, + it have yJson object which contain all parsed YANG models + + Returns: + list - list 'grouping' entities + """ + ret_grouping = list() + + for i in range(len(conf_mgmt.sy.yJson)): + if (conf_mgmt.sy.yJson[i].get('module').get('@name') == yang_model_name): + grouping = conf_mgmt.sy.yJson[i].get('module').get('grouping') + if isinstance(grouping, list): + for group in grouping: + ret_grouping.append(group) + else: + ret_grouping.append(grouping) + + return ret_grouping + +def get_import_prefixes(y_uses: OrderedDict) -> list: + """ Parse 'import prefix' of YANG 'uses' entity + Example: + { + uses stypes:endpoint; + } + 'stypes' - prefix of imported YANG module. + 'endpoint' - YANG 'grouping' entity name + + Args: + y_uses: refrence to YANG 'uses' + Returns: + list - of parsed prefixes + """ + ret_prefixes = list() + + if isinstance(y_uses, list): + for use in y_uses: + prefix = use.get('@name').split(':')[0] + if prefix != use.get('@name'): + ret_prefixes.append(prefix) + else: + prefix = y_uses.get('@name').split(':')[0] + if prefix != y_uses.get('@name'): + ret_prefixes.append(prefix) + + return ret_prefixes + def get_list_keys(y_list: OrderedDict) -> list: ret_list = list() keys = y_list.get('key').get('@value').split() diff --git a/tests/cli_autogen_yang_parser_test.py b/tests/cli_autogen_yang_parser_test.py index acabfe5082..1e61b00fba 100644 --- a/tests/cli_autogen_yang_parser_test.py +++ b/tests/cli_autogen_yang_parser_test.py @@ -17,47 +17,47 @@ class TestYangParser: - #def test_1_table_container(self): - # yang_model_name = 'sonic-1-table-container' - # template(yang_model_name, assert_dictionaries.one_table_container) - # - #def test_2_table_containers(self): - # yang_model_name = 'sonic-2-table-containers' - # template(yang_model_name, assert_dictionaries.two_table_containers) - - #def test_1_object_container(self): - # yang_model_name = 'sonic-1-object-container' - # template(yang_model_name, assert_dictionaries.one_object_container) - - #def test_2_object_containers(self): - # yang_model_name = 'sonic-2-object-containers' - # template(yang_model_name, assert_dictionaries.two_object_containers) - - #def test_1_list(self): - # yang_model_name = 'sonic-1-list' - # template(yang_model_name, assert_dictionaries.one_list) - - #def test_2_lists(self): - # yang_model_name = 'sonic-2-lists' - # template(yang_model_name, assert_dictionaries.two_lists) - - #def test_static_object_complex_1(self): - # """ Test object container with: 1 leaf, 1 leaf-list, 1 choice. - # """ - # yang_model_name = 'sonic-static-object-complex-1' - # template(yang_model_name, assert_dictionaries.static_object_complex_1) - - #def test_static_object_complex_2(self): - # """ Test object container with: 2 leafs, 2 leaf-lists, 2 choices. - # """ - # yang_model_name = 'sonic-static-object-complex-2' - # template(yang_model_name, assert_dictionaries.static_object_complex_2) - - #def test_dynamic_object_complex_1(self): - # """ Test object container with: 1 key, 1 leaf, 1 leaf-list, 1 choice. - # """ - # yang_model_name = 'sonic-dynamic-object-complex-1' - # template(yang_model_name, assert_dictionaries.dynamic_object_complex_1) + def test_1_table_container(self): + yang_model_name = 'sonic-1-table-container' + template(yang_model_name, assert_dictionaries.one_table_container) + + def test_2_table_containers(self): + yang_model_name = 'sonic-2-table-containers' + template(yang_model_name, assert_dictionaries.two_table_containers) + + def test_1_object_container(self): + yang_model_name = 'sonic-1-object-container' + template(yang_model_name, assert_dictionaries.one_object_container) + + def test_2_object_containers(self): + yang_model_name = 'sonic-2-object-containers' + template(yang_model_name, assert_dictionaries.two_object_containers) + + def test_1_list(self): + yang_model_name = 'sonic-1-list' + template(yang_model_name, assert_dictionaries.one_list) + + def test_2_lists(self): + yang_model_name = 'sonic-2-lists' + template(yang_model_name, assert_dictionaries.two_lists) + + def test_static_object_complex_1(self): + """ Test object container with: 1 leaf, 1 leaf-list, 1 choice. + """ + yang_model_name = 'sonic-static-object-complex-1' + template(yang_model_name, assert_dictionaries.static_object_complex_1) + + def test_static_object_complex_2(self): + """ Test object container with: 2 leafs, 2 leaf-lists, 2 choices. + """ + yang_model_name = 'sonic-static-object-complex-2' + template(yang_model_name, assert_dictionaries.static_object_complex_2) + + def test_dynamic_object_complex_1(self): + """ Test object container with: 1 key, 1 leaf, 1 leaf-list, 1 choice. + """ + yang_model_name = 'sonic-dynamic-object-complex-1' + template(yang_model_name, assert_dictionaries.dynamic_object_complex_1) def test_dynamic_object_complex_2(self): """ Test object container with: 2 keys, 2 leafs, 2 leaf-list, 2 choice. @@ -65,6 +65,10 @@ def test_dynamic_object_complex_2(self): yang_model_name = 'sonic-dynamic-object-complex-2' template(yang_model_name, assert_dictionaries.dynamic_object_complex_2) + # TODO: UT for choice + # TODO: UT for grouping + + def template(yang_model_name, correct_dict): config_db_path = os.path.join(test_path, 'cli_autogen_input/config_db.json') move_yang_model(yang_model_name) From b0fb3a96ff9cc087f89fe49421604a036d34e0b7 Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Tue, 18 May 2021 15:39:55 +0000 Subject: [PATCH 122/173] Added 'group' to 'attrs' Signed-off-by: Vadym Hlushko --- sonic_cli_gen/yang_parser.py | 35 +++++++------- .../cli_autogen_input/assert_dictionaries.py | 24 ++++++++++ .../sonic-choice-complex-1.yang | 47 +++++++++++++++++++ tests/cli_autogen_input/sonic-grouping.yang | 18 +++++++ tests/cli_autogen_yang_parser_test.py | 8 ++++ 5 files changed, 115 insertions(+), 17 deletions(-) create mode 100644 tests/cli_autogen_input/sonic-choice-complex-1.yang create mode 100644 tests/cli_autogen_input/sonic-grouping.yang diff --git a/sonic_cli_gen/yang_parser.py b/sonic_cli_gen/yang_parser.py index bb05d5a21a..4a8664b78e 100644 --- a/sonic_cli_gen/yang_parser.py +++ b/sonic_cli_gen/yang_parser.py @@ -206,8 +206,8 @@ def on_object_container(y_module: OrderedDict, cont: OrderedDict, conf_mgmt, is_ obj_elem['keys'] = get_list_keys(cont) attrs_list = list() - attrs_list.extend(get_leafs(cont)) - attrs_list.extend(get_leaf_lists(cont)) + attrs_list.extend(get_leafs(cont, grouping_name = '')) + attrs_list.extend(get_leaf_lists(cont, grouping_name = '')) attrs_list.extend(get_choices(y_module, cont, conf_mgmt)) # TODO: need to test 'grouping' attrs_list.extend(get_uses(y_module, cont, conf_mgmt)) @@ -237,13 +237,13 @@ def on_uses(y_module: OrderedDict, y_uses, conf_mgmt) -> list: if isinstance(y_uses, list): for use in y_uses: if use.get('@name') == group.get('@name'): - ret_attrs.extend(get_leafs(group)) - ret_attrs.extend(get_leaf_lists(group)) + ret_attrs.extend(get_leafs(group, group.get('@name'))) + ret_attrs.extend(get_leaf_lists(group, grouping_name = '')) ret_attrs.extend(get_choices(y_module, group, conf_mgmt)) else: if y_uses.get('@name') == group.get('@name'): - ret_attrs.extend(get_leafs(group)) - ret_attrs.extend(get_leaf_lists(group)) + ret_attrs.extend(get_leafs(group, group.get('@name'))) + ret_attrs.extend(get_leaf_lists(group, grouping_name = '')) ret_attrs.extend(get_choices(y_module, group, conf_mgmt)) return ret_attrs @@ -284,8 +284,8 @@ def on_choice_cases(y_module: OrderedDict, y_cases: list, conf_mgmt) -> list: if isinstance(y_cases, list): for case in y_cases: - ret_attrs.extend(get_leafs(case)) - ret_attrs.extend(get_leaf_lists(case)) + ret_attrs.extend(get_leafs(case, grouping_name = '')) + ret_attrs.extend(get_leaf_lists(case, grouping_name = '')) # TODO: need to deeply test it ret_attrs.extend(get_uses(y_module, case, conf_mgmt)) else: @@ -293,7 +293,7 @@ def on_choice_cases(y_module: OrderedDict, y_cases: list, conf_mgmt) -> list: return ret_attrs -def on_leafs(y_leafs, is_leaf_list: bool) -> list: +def on_leafs(y_leafs, grouping_name, is_leaf_list: bool) -> list: """ Parse all the 'leaf' or 'leaf-list' elements Args: @@ -306,15 +306,15 @@ def on_leafs(y_leafs, is_leaf_list: bool) -> list: # The YANG 'container' entity may have only 1 'leaf' element OR a list of 'leaf' elements if isinstance(y_leafs, list): for leaf in y_leafs: - attr = on_leaf(leaf, is_leaf_list) + attr = on_leaf(leaf, is_leaf_list, grouping_name) ret_attrs.append(attr) else: - attr = on_leaf(y_leafs, is_leaf_list) + attr = on_leaf(y_leafs, is_leaf_list, grouping_name) ret_attrs.append(attr) return ret_attrs -def on_leaf(leaf: OrderedDict, is_leaf_list: bool) -> dict: +def on_leaf(leaf: OrderedDict, is_leaf_list: bool, grouping_name: str) -> dict: """ Parse a single 'leaf' element Args: @@ -326,7 +326,8 @@ def on_leaf(leaf: OrderedDict, is_leaf_list: bool) -> dict: attr = { 'name': leaf.get('@name'), 'description': get_description(leaf), 'is-leaf-list': is_leaf_list, - 'is-mandatory': get_mandatory(leaf) } + 'is-mandatory': get_mandatory(leaf), + 'group': grouping_name} return attr @@ -352,15 +353,15 @@ def get_description(y_entity: OrderedDict) -> str: else: return '' -def get_leafs(y_entity: OrderedDict) -> list: +def get_leafs(y_entity: OrderedDict, grouping_name) -> list: if y_entity.get('leaf') is not None: - return on_leafs(y_entity.get('leaf'), is_leaf_list=False) + return on_leafs(y_entity.get('leaf'), grouping_name, is_leaf_list=False) return [] -def get_leaf_lists(y_entity: OrderedDict) -> list: +def get_leaf_lists(y_entity: OrderedDict, grouping_name) -> list: if y_entity.get('leaf-list') is not None: - return on_leafs(y_entity.get('leaf-list'), is_leaf_list=True) + return on_leafs(y_entity.get('leaf-list'), grouping_name, is_leaf_list=True) return [] diff --git a/tests/cli_autogen_input/assert_dictionaries.py b/tests/cli_autogen_input/assert_dictionaries.py index 3aac3c2f6f..751784dae9 100644 --- a/tests/cli_autogen_input/assert_dictionaries.py +++ b/tests/cli_autogen_input/assert_dictionaries.py @@ -151,24 +151,28 @@ "description": "OBJ_1_LEAF_1 description", "is-mandatory": False, "is-leaf-list": False, + "group": '', }, { "name":"OBJ_1_LEAF_LIST_1", "description": "", "is-mandatory": False, "is-leaf-list": True, + "group": '', }, { "name":"OBJ_1_CHOICE_1_LEAF_1", "description": "", "is-mandatory": False, "is-leaf-list": False, + "group": '', }, { "name":"OBJ_1_CHOICE_1_LEAF_2", "description": "", "is-mandatory": False, "is-leaf-list": False, + "group": '', } ] } @@ -192,48 +196,56 @@ "description": "OBJ_1_LEAF_1 description", "is-mandatory": False, "is-leaf-list": False, + "group": '', }, { "name":"OBJ_1_LEAF_2", "description": "OBJ_1_LEAF_2 description", "is-mandatory": False, "is-leaf-list": False, + "group": '', }, { "name":"OBJ_1_LEAF_LIST_1", "description": "", "is-mandatory": False, "is-leaf-list": True, + "group": '', }, { "name":"OBJ_1_LEAF_LIST_2", "description": "", "is-mandatory": False, "is-leaf-list": True, + "group": '', }, { "name":"OBJ_1_CHOICE_1_LEAF_1", "description": "", "is-mandatory": False, "is-leaf-list": False, + "group": '', }, { "name":"OBJ_1_CHOICE_1_LEAF_2", "description": "", "is-mandatory": False, "is-leaf-list": False, + "group": '', }, { "name":"OBJ_1_CHOICE_2_LEAF_1", "description": "", "is-mandatory": False, "is-leaf-list": False, + "group": '', }, { "name":"OBJ_1_CHOICE_2_LEAF_2", "description": "", "is-mandatory": False, "is-leaf-list": False, + "group": '', }, ] } @@ -257,24 +269,28 @@ "description": "OBJ_1_LEAF_1 description", "is-mandatory": False, "is-leaf-list": False, + "group": '', }, { "name":"OBJ_1_LEAF_LIST_1", "description": "", "is-mandatory": False, "is-leaf-list": True, + "group": '', }, { "name":"OBJ_1_CHOICE_1_LEAF_1", "description": "", "is-mandatory": False, "is-leaf-list": False, + "group": '', }, { "name":"OBJ_1_CHOICE_1_LEAF_2", "description": "", "is-mandatory": False, "is-leaf-list": False, + "group": '', } ], "keys":[ @@ -304,48 +320,56 @@ "description": "OBJ_1_LEAF_1 description", "is-mandatory": False, "is-leaf-list": False, + "group": '', }, { "name":"OBJ_1_LEAF_2", "description": "OBJ_1_LEAF_2 description", "is-mandatory": False, "is-leaf-list": False, + "group": '', }, { "name":"OBJ_1_LEAF_LIST_1", "description": "", "is-mandatory": False, "is-leaf-list": True, + "group": '', }, { "name":"OBJ_1_LEAF_LIST_2", "description": "", "is-mandatory": False, "is-leaf-list": True, + "group": '', }, { "name":"OBJ_1_CHOICE_1_LEAF_1", "description": "", "is-mandatory": False, "is-leaf-list": False, + "group": '', }, { "name":"OBJ_1_CHOICE_1_LEAF_2", "description": "", "is-mandatory": False, "is-leaf-list": False, + "group": '', }, { "name":"OBJ_1_CHOICE_2_LEAF_1", "description": "", "is-mandatory": False, "is-leaf-list": False, + "group": '', }, { "name":"OBJ_1_CHOICE_2_LEAF_2", "description": "", "is-mandatory": False, "is-leaf-list": False, + "group": '', } ], "keys":[ diff --git a/tests/cli_autogen_input/sonic-choice-complex-1.yang b/tests/cli_autogen_input/sonic-choice-complex-1.yang new file mode 100644 index 0000000000..4e45717493 --- /dev/null +++ b/tests/cli_autogen_input/sonic-choice-complex-1.yang @@ -0,0 +1,47 @@ +module sonic-static-object-complex-1 { + + yang-version 1.1; + + namespace "http://github.com/Azure/static-complex-1"; + prefix static-complex-1; + + import sonic-grouping { + prefix sgrop; + } + + container sonic-static-object-complex-1 { + /* sonic-static-object-complex-1 - top level container */ + + container TABLE_1 { + /* TABLE_1 - table container */ + + description "TABLE_1 description"; + + container OBJECT_1 { + /* OBJECT_1 - object container, it have: + * 1 choice + */ + + description "OBJECT_1 description"; + + choice CHOICE_1 { + case CHOICE_1_CASE_1 { + leaf LEAF_1 { + type uint16; + } + + leaf-list LEAF_LIST_1 { + type string; + } + } + + case CHOICE_1_CASE_2 { + leaf LEAF_2 { + type string; + } + } + } + } + } + } +} \ No newline at end of file diff --git a/tests/cli_autogen_input/sonic-grouping.yang b/tests/cli_autogen_input/sonic-grouping.yang new file mode 100644 index 0000000000..59aa0b5948 --- /dev/null +++ b/tests/cli_autogen_input/sonic-grouping.yang @@ -0,0 +1,18 @@ +module sonic-grouping{ + + yang-version 1.1; + + namespace "http://github.com/Azure/s-grouping"; + prefix s-grouping; + + grouping target { + leaf t_address { + type inet:ip-address; + description "Target IP address."; + } + leaf t_port { + type inet:port-number; + description "Target port number."; + } + } +} \ No newline at end of file diff --git a/tests/cli_autogen_yang_parser_test.py b/tests/cli_autogen_yang_parser_test.py index 1e61b00fba..647f9e1907 100644 --- a/tests/cli_autogen_yang_parser_test.py +++ b/tests/cli_autogen_yang_parser_test.py @@ -65,6 +65,14 @@ def test_dynamic_object_complex_2(self): yang_model_name = 'sonic-dynamic-object-complex-2' template(yang_model_name, assert_dictionaries.dynamic_object_complex_2) + #def test_choice_complex_1(self): + # """ Test object container with choice that have: 1 leaf, 1 leaf-list, 1 uses + # """ + # yang_model_name = 'sonic-choice-complex' + # grouping_yang = 'sonic-grouping' + # move_yang_model(grouping_yang) + # template(yang_model_name, assert_dictionaries.dynamic_object_complex_2) + # TODO: UT for choice # TODO: UT for grouping From 2008cbc2e52991fee572132210faff29e4793199 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Tue, 18 May 2021 19:07:42 +0300 Subject: [PATCH 123/173] [sonic-package-manager] update feature configuration in case it requires changes Signed-off-by: Stepan Blyschak --- sonic_package_manager/dockerapi.py | 10 ++++++ sonic_package_manager/manager.py | 30 ++++++++-------- .../service_creator/feature.py | 30 ++++++++++++++++ .../test_service_creator.py | 36 +++++++++++++++++++ 4 files changed, 91 insertions(+), 15 deletions(-) diff --git a/sonic_package_manager/dockerapi.py b/sonic_package_manager/dockerapi.py index 926600d0bc..a0aaabf430 100644 --- a/sonic_package_manager/dockerapi.py +++ b/sonic_package_manager/dockerapi.py @@ -185,6 +185,16 @@ def rm(self, container: str, **kwargs): self.client.containers.get(container).remove(**kwargs) log.debug(f'removed container {container}') + + def rm_by_ancestor(self, image_id: str, **kwargs): + """ Docker 'rm' command for running containers instantiated + from image passed to this function. """ + + # Clean containers based on the old image + containers = self.ps(filters={'ancestor': image_id}, + all=True) + for container in containers: + self.rm(container.id, **kwargs) def ps(self, **kwargs): """ Docker 'ps' command. """ diff --git a/sonic_package_manager/manager.py b/sonic_package_manager/manager.py index 9eed20321e..d968f11c49 100644 --- a/sonic_package_manager/manager.py +++ b/sonic_package_manager/manager.py @@ -2,6 +2,7 @@ import contextlib import functools +from genericpath import exists import os import pkgutil import tempfile @@ -481,13 +482,7 @@ def uninstall(self, name: str, self.service_creator.generate_shutdown_sequence_files( self._get_installed_packages_except(package) ) - - # Clean containers based on this image - containers = self.docker.ps(filters={'ancestor': package.image_id}, - all=True) - for container in containers: - self.docker.rm(container.id, force=True) - + self.docker.rm_by_ancestor(package.image_id, force=True) self.docker.rmi(package.image_id, force=True) package.entry.image_id = None except Exception as err: @@ -566,7 +561,6 @@ def upgrade_from_source(self, service_create_opts = { 'register_feature': False, } - service_remove_opts = { 'deregister_feature': False, } @@ -579,7 +573,9 @@ def upgrade_from_source(self, source.install(new_package) exits.callback(rollback(source.uninstall, new_package)) - if self.feature_registry.is_feature_enabled(old_feature): + feature_enabled = self.feature_registry.is_feature_enabled(old_feature) + + if feature_enabled: self._systemctl_action(old_package, 'stop') exits.callback(rollback(self._systemctl_action, old_package, 'start')) @@ -588,11 +584,7 @@ def upgrade_from_source(self, exits.callback(rollback(self.service_creator.create, old_package, **service_create_opts)) - # Clean containers based on the old image - containers = self.docker.ps(filters={'ancestor': old_package.image_id}, - all=True) - for container in containers: - self.docker.rm(container.id, force=True) + self.docker.rm_by_ancestor(old_package.image_id, force=True) self.service_creator.create(new_package, **service_create_opts) exits.callback(rollback(self.service_creator.remove, new_package, @@ -606,11 +598,19 @@ def upgrade_from_source(self, self._get_installed_packages_and(old_package)) ) - if self.feature_registry.is_feature_enabled(new_feature): + if feature_enabled: self._systemctl_action(new_package, 'start') exits.callback(rollback(self._systemctl_action, new_package, 'stop')) + # Update feature configuration after we have started new service. + # If we place it before the above, we our service start/stop will + # interfier with hostcfgd in rollback path leading to + self.feature_registry.update(old_package.manifest, new_package.manifest) + exits.callback(rollback( + self.feature_registry.update, new_package.manifest, old_package.manifest) + ) + if not skip_host_plugins: self._install_cli_plugins(new_package) exits.callback(rollback(self._uninstall_cli_plugin, old_package)) diff --git a/sonic_package_manager/service_creator/feature.py b/sonic_package_manager/service_creator/feature.py index dbc93a8284..e93ddfd44f 100644 --- a/sonic_package_manager/service_creator/feature.py +++ b/sonic_package_manager/service_creator/feature.py @@ -73,6 +73,36 @@ def deregister(self, name: str): db_connetors = self._sonic_db.get_connectors() for conn in db_connetors: conn.set_entry(FEATURE, name, None) + + def update(self, + old_manifest: Manifest, + new_manifest: Manifest): + """ Migrate feature configuration. It can be that non-configurable + feature entries have to be updated. e.g: "has_timer" for example if + the new feature introduces a service timer or name of the service has + changed, but user configurable entries are not changed). + + Args: + old_manifest: Old feature manifest. + new_manifest: New feature manifest. + Returns: + None + """ + + old_name = old_manifest['service']['name'] + new_name = new_manifest['service']['name'] + db_connectors = self._sonic_db.get_connectors() + non_cfg_entries = self.get_non_configurable_feature_entries(new_manifest) + + for conn in db_connectors: + current_cfg = conn.get_entry(FEATURE, old_name) + conn.set_entry(FEATURE, old_name, None) + + new_cfg = current_cfg.copy() + # Override CONFIG DB data with non configurable entries. + new_cfg = {**new_cfg, **non_cfg_entries} + + conn.set_entry(FEATURE, new_name, new_cfg) def is_feature_enabled(self, name: str) -> bool: """ Returns whether the feature is current enabled diff --git a/tests/sonic_package_manager/test_service_creator.py b/tests/sonic_package_manager/test_service_creator.py index 9bfac219d4..be92171b3f 100644 --- a/tests/sonic_package_manager/test_service_creator.py +++ b/tests/sonic_package_manager/test_service_creator.py @@ -1,6 +1,7 @@ #!/usr/bin/env python import os +import copy from unittest.mock import Mock, call import pytest @@ -222,6 +223,41 @@ def test_feature_registration(mock_sonic_db, manifest): }) +def test_feature_update(mock_sonic_db, manifest): + curr_feature_config = { + 'state': 'enabled', + 'auto_restart': 'enabled', + 'high_mem_alert': 'disabled', + 'set_owner': 'local', + 'has_per_asic_scope': 'False', + 'has_global_scope': 'True', + 'has_timer': 'False', + } + mock_connector = Mock() + mock_connector.get_entry = Mock(return_value=curr_feature_config) + mock_sonic_db.get_connectors = Mock(return_value=[mock_connector]) + feature_registry = FeatureRegistry(mock_sonic_db) + + new_manifest = copy.deepcopy(manifest) + new_manifest['service']['name'] = 'test_new' + new_manifest['service']['delayed'] = True + + feature_registry.update(manifest, new_manifest) + + mock_connector.set_entry.assert_has_calls([ + call('FEATURE', 'test', None), + call('FEATURE', 'test_new', { + 'state': 'enabled', + 'auto_restart': 'enabled', + 'high_mem_alert': 'disabled', + 'set_owner': 'local', + 'has_per_asic_scope': 'False', + 'has_global_scope': 'True', + 'has_timer': 'True', + }), + ], any_order=True) + + def test_feature_registration_with_timer(mock_sonic_db, manifest): manifest['service']['delayed'] = True mock_connector = Mock() From a282872ebb090da48c6638abae0ef8da871228f4 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Wed, 19 May 2021 12:25:23 +0300 Subject: [PATCH 124/173] Improvements and fixes Signed-off-by: Stepan Blyschak --- sonic_package_manager/constraint.py | 2 +- sonic_package_manager/dockerapi.py | 5 +- sonic_package_manager/errors.py | 1 - sonic_package_manager/main.py | 12 ++-- sonic_package_manager/manager.py | 60 ++++++++++--------- sonic_package_manager/metadata.py | 6 +- sonic_package_manager/registry.py | 2 +- .../service_creator/creator.py | 58 +++++++++--------- .../service_creator/feature.py | 14 ++--- .../service_creator/sonic_db.py | 9 +-- 10 files changed, 83 insertions(+), 86 deletions(-) diff --git a/sonic_package_manager/constraint.py b/sonic_package_manager/constraint.py index af5a13000b..70b7165354 100644 --- a/sonic_package_manager/constraint.py +++ b/sonic_package_manager/constraint.py @@ -46,7 +46,7 @@ def parse(constraints: Dict) -> 'ComponentConstraints': """ components = {component: VersionConstraint.parse(version) - for component, version in constraints.items()} + for component, version in constraints.items()} return ComponentConstraints(components) def deparse(self) -> Dict[str, str]: diff --git a/sonic_package_manager/dockerapi.py b/sonic_package_manager/dockerapi.py index a0aaabf430..7f051d2d72 100644 --- a/sonic_package_manager/dockerapi.py +++ b/sonic_package_manager/dockerapi.py @@ -185,14 +185,13 @@ def rm(self, container: str, **kwargs): self.client.containers.get(container).remove(**kwargs) log.debug(f'removed container {container}') - + def rm_by_ancestor(self, image_id: str, **kwargs): """ Docker 'rm' command for running containers instantiated from image passed to this function. """ # Clean containers based on the old image - containers = self.ps(filters={'ancestor': image_id}, - all=True) + containers = self.ps(filters={'ancestor': image_id}, all=True) for container in containers: self.rm(container.id, **kwargs) diff --git a/sonic_package_manager/errors.py b/sonic_package_manager/errors.py index 17279c52c4..fe4de39a39 100644 --- a/sonic_package_manager/errors.py +++ b/sonic_package_manager/errors.py @@ -143,4 +143,3 @@ class PackageComponentConflictError(PackageInstallationError): def __str__(self): return (f'Package {self.name} conflicts with {self.component} {self.constraint} ' f'in package {self.dependency} but version {self.installed_ver} is installed') - diff --git a/sonic_package_manager/main.py b/sonic_package_manager/main.py index 9034da7fd7..8a0aabb901 100644 --- a/sonic_package_manager/main.py +++ b/sonic_package_manager/main.py @@ -361,7 +361,7 @@ def install(ctx, package_source = package_expr or from_repository or from_tarball if not package_source: - exit_cli(f'Package source is not specified', fg='red') + exit_cli('Package source is not specified', fg='red') if not yes and not force: click.confirm(f'{package_source} is going to be installed, ' @@ -386,7 +386,7 @@ def install(ctx, except Exception as err: exit_cli(f'Failed to install {package_source}: {err}', fg='red') except KeyboardInterrupt: - exit_cli(f'Operation canceled by user', fg='red') + exit_cli('Operation canceled by user', fg='red') @cli.command() @@ -409,7 +409,7 @@ def reset(ctx, name, force, yes, skip_host_plugins): except Exception as err: exit_cli(f'Failed to reset package {name}: {err}', fg='red') except KeyboardInterrupt: - exit_cli(f'Operation canceled by user', fg='red') + exit_cli('Operation canceled by user', fg='red') @cli.command() @@ -426,7 +426,7 @@ def uninstall(ctx, name, force, yes, keep_config): if not yes and not force: click.confirm(f'Package {name} is going to be uninstalled, ' f'continue?', abort=True, show_default=True) - + uninstall_opts = { 'force': force, 'keep_config': keep_config, @@ -437,7 +437,7 @@ def uninstall(ctx, name, force, yes, keep_config): except Exception as err: exit_cli(f'Failed to uninstall package {name}: {err}', fg='red') except KeyboardInterrupt: - exit_cli(f'Operation canceled by user', fg='red') + exit_cli('Operation canceled by user', fg='red') @cli.command() @@ -459,7 +459,7 @@ def migrate(ctx, database, force, yes, dockerd_socket): except Exception as err: exit_cli(f'Failed to migrate packages {err}', fg='red') except KeyboardInterrupt: - exit_cli(f'Operation canceled by user', fg='red') + exit_cli('Operation canceled by user', fg='red') if __name__ == "__main__": diff --git a/sonic_package_manager/manager.py b/sonic_package_manager/manager.py index d968f11c49..571a73a00f 100644 --- a/sonic_package_manager/manager.py +++ b/sonic_package_manager/manager.py @@ -2,7 +2,6 @@ import contextlib import functools -from genericpath import exists import os import pkgutil import tempfile @@ -162,7 +161,7 @@ def get_cli_plugin_directory(command: str) -> str: def get_cli_plugin_path(package: Package, command: str) -> str: """ Returns a path where to put CLI plugin code. - + Args: package: Package to generate this path for. command: SONiC command: "show"/"config"/"clear". @@ -251,11 +250,10 @@ def validate_package_tree(packages: Dict[str, Package]): continue component_version = conflicting_package.components[component] - log.debug(f'conflicting package {dependency.name}: ' + log.debug(f'conflicting package {conflict.name}: ' f'component {component} version is {component_version}') - if constraint.allows_all(component_version): - raise PackageComponentConflictError(package.name, dependency, component, + raise PackageComponentConflictError(package.name, conflict, component, constraint, component_version) @@ -400,7 +398,7 @@ def install_from_source(self, # package name may not be in database. if not self.database.has_package(package.name): self.database.add_package(package.name, package.repository) - + service_create_opts = { 'state': feature_state, 'owner': default_owner, @@ -557,7 +555,7 @@ def upgrade_from_source(self, validate_package_cli_can_be_skipped(new_package, skip_host_plugins) # After all checks are passed we proceed to actual upgrade - + service_create_opts = { 'register_feature': False, } @@ -575,12 +573,15 @@ def upgrade_from_source(self, feature_enabled = self.feature_registry.is_feature_enabled(old_feature) - if feature_enabled: + if feature_enabled: + self._systemctl_action(new_package, 'disable') + exits.callback(rollback(self._systemctl_action, + old_package, 'enable')) self._systemctl_action(old_package, 'stop') exits.callback(rollback(self._systemctl_action, old_package, 'start')) - self.service_creator.remove(old_package, **service_remove_opts) + self.service_creator.remove(old_package, **service_remove_opts) exits.callback(rollback(self.service_creator.create, old_package, **service_create_opts)) @@ -598,14 +599,18 @@ def upgrade_from_source(self, self._get_installed_packages_and(old_package)) ) - if feature_enabled: + if feature_enabled: + self._systemctl_action(new_package, 'enable') + exits.callback(rollback(self._systemctl_action, + old_package, 'disable')) self._systemctl_action(new_package, 'start') exits.callback(rollback(self._systemctl_action, new_package, 'stop')) # Update feature configuration after we have started new service. - # If we place it before the above, we our service start/stop will - # interfier with hostcfgd in rollback path leading to + # If we place it before the above, our service start/stop will + # interfere with hostcfgd in rollback path leading to a service + # running with new image and not the old one. self.feature_registry.update(old_package.manifest, new_package.manifest) exits.callback(rollback( self.feature_registry.update, new_package.manifest, old_package.manifest) @@ -662,16 +667,16 @@ def migrate_packages(self, old_package_database: PackageDatabase, dockerd_sock: Optional[str] = None): """ - Migrate packages from old database. This function can do a comparison between - current database and the database passed in as argument. If the package is - missing in the current database it will be added. If the package is installed - in the passed database and in the current it is not installed it will be - installed with a passed database package version. If the package is installed - in the passed database and it is installed in the current database but with - older version the package will be upgraded to the never version. If the package - is installed in the passed database and in the current it is installed but with - never version - no actions are taken. If dockerd_sock parameter is passed, the - migration process will use loaded images from docker library of the currently + Migrate packages from old database. This function can do a comparison between + current database and the database passed in as argument. If the package is + missing in the current database it will be added. If the package is installed + in the passed database and in the current it is not installed it will be + installed with a passed database package version. If the package is installed + in the passed database and it is installed in the current database but with + older version the package will be upgraded to the never version. If the package + is installed in the passed database and in the current it is installed but with + never version - no actions are taken. If dockerd_sock parameter is passed, the + migration process will use loaded images from docker library of the currently installed image. Args: @@ -823,8 +828,8 @@ def get_package_source(self, if package_entry.default_reference is not None: package_ref.reference = package_entry.default_reference else: - raise PackageManagerError(f'No default reference tag. ' - f'Please specify the version or tag explicitly') + raise PackageManagerError('No default reference tag. ' + 'Please specify the version or tag explicitly') return RegistrySource(package_entry.repository, package_ref.reference, @@ -896,7 +901,7 @@ def get_installed_packages_list(self) -> List[Package]: Installed packages dictionary. """ - return [self.get_installed_package(entry.name) + return [self.get_installed_package(entry.name) for entry in self.database if entry.installed] def _migrate_package_database(self, old_package_database: PackageDatabase): @@ -991,10 +996,9 @@ def get_manager() -> 'PackageManager': metadata_resolver = MetadataResolver(docker_api, registry_resolver) cfg_mgmt = config_mgmt.ConfigMgmt() cli_generator = CliGenerator() - sonic_db = SonicDB() - feature_registry = FeatureRegistry(sonic_db) + feature_registry = FeatureRegistry(SonicDB) service_creator = ServiceCreator(feature_registry, - sonic_db, + SonicDB, cli_generator, cfg_mgmt) diff --git a/sonic_package_manager/metadata.py b/sonic_package_manager/metadata.py index 45cca67642..3475cf2c70 100644 --- a/sonic_package_manager/metadata.py +++ b/sonic_package_manager/metadata.py @@ -24,10 +24,10 @@ def deep_update(dst: Dict, src: Dict) -> Dict: for key, value in src.items(): if isinstance(value, dict): - node = dst.setdefault(key, {}) - deep_update(node, value) + node = dst.setdefault(key, {}) + deep_update(node, value) else: - dst[key] = value + dst[key] = value return dst diff --git a/sonic_package_manager/registry.py b/sonic_package_manager/registry.py index 8a09d9136e..8c03b078d2 100644 --- a/sonic_package_manager/registry.py +++ b/sonic_package_manager/registry.py @@ -38,7 +38,7 @@ def get_token(realm, service, scope) -> str: response = requests.get(f'{realm}?scope={scope}&service={service}') if response.status_code != requests.codes.ok: - raise AuthenticationServiceError(f'Failed to retrieve token') + raise AuthenticationServiceError('Failed to retrieve token') content = json.loads(response.content) token = content['token'] diff --git a/sonic_package_manager/service_creator/creator.py b/sonic_package_manager/service_creator/creator.py index 28b47e5a54..3b1464ffa2 100644 --- a/sonic_package_manager/service_creator/creator.py +++ b/sonic_package_manager/service_creator/creator.py @@ -12,7 +12,7 @@ from prettyprinter import pformat from toposort import toposort_flatten, CircularDependencyError -from utilities_common.general import load_module_from_source +from config.config_mgmt import sonic_cfggen from sonic_cli_gen.generator import CliGenerator from sonic_package_manager.logger import log @@ -25,8 +25,6 @@ from sonic_package_manager.service_creator.sonic_db import SonicDB from sonic_package_manager.service_creator.utils import in_chroot -# Load sonic-cfggen from source since /usr/local/bin/sonic-cfggen does not have .py extension. -sonic_cfggen = load_module_from_source('sonic_cfggen', '/usr/local/bin/sonic-cfggen') SERVICE_FILE_TEMPLATE = 'sonic.service.j2' TIMER_UNIT_TEMPLATE = 'timer.unit.j2' @@ -129,7 +127,7 @@ def __init__(self, cli_gen: CliGenerator, cfg_mgmt: ConfigMgmt): """ Initialize ServiceCreator with: - + Args: feature_registry: FeatureRegistry object. sonic_db: SonicDB interface. @@ -146,8 +144,8 @@ def create(self, register_feature: bool = True, state: str = 'enabled', owner: str = 'local'): - """ Register package as SONiC service. - + """ Register package as SONiC service. + Args: package: Package object to install. register_feature: Wether to register this package in FEATURE table. @@ -181,7 +179,7 @@ def remove(self, deregister_feature: bool = True, keep_config: bool = False): """ Uninstall SONiC service provided by the package. - + Args: package: Package object to uninstall. deregister_feature: Wether to deregister this package from FEATURE table. @@ -211,8 +209,8 @@ def remove(self, self.feature_registry.deregister(package.manifest['service']['name']) def generate_container_mgmt(self, package: Package): - """ Generates container management script under /usr/bin/.sh for package. - + """ Generates container management script under /usr/bin/.sh for package. + Args: package: Package object to generate script for. Returns: @@ -254,8 +252,8 @@ def generate_container_mgmt(self, package: Package): log.info(f'generated {script_path}') def generate_service_mgmt(self, package: Package): - """ Generates service management script under /usr/local/bin/.sh for package. - + """ Generates service management script under /usr/local/bin/.sh for package. + Args: package: Package object to generate script for. Returns: @@ -275,8 +273,8 @@ def generate_service_mgmt(self, package: Package): log.info(f'generated {script_path}') def generate_systemd_service(self, package: Package): - """ Generates systemd service(s) file and timer(s) (if needed) for package. - + """ Generates systemd service(s) file and timer(s) (if needed) for package. + Args: package: Package object to generate service for. Returns: @@ -323,13 +321,13 @@ def generate_systemd_service(self, package: Package): def update_dependent_list_file(self, package: Package, remove=False): """ This function updates dependent list file for packages listed in "dependent-of" (path: /etc/sonic/_dependent file). - + Args: package: Package to update packages dependent of it. Returns: None. """ - + name = package.manifest['service']['name'] dependent_of = package.manifest['service']['dependent-of'] host_service = package.manifest['service']['host-service'] @@ -363,7 +361,7 @@ def update_dependent(service, name, multi_inst): def generate_dump_script(self, package): """ Generates dump plugin script for package. - + Args: package: Package object to generate dump plugin script for. Returns: @@ -389,7 +387,7 @@ def generate_dump_script(self, package): def get_shutdown_sequence(self, reboot_type: str, packages: Dict[str, Package]): """ Returns shutdown sequence file for particular reboot type. - + Args: reboot_type: Reboot type to generated service shutdown sequence for. packages: Dict of installed packages. @@ -436,7 +434,7 @@ def filter_not_available(services): def generate_shutdown_sequence_file(self, reboot_type: str, packages: Dict[str, Package]): """ Generates shutdown sequence file for particular reboot type (path: /etc/sonic/-reboot_order). - + Args: reboot_type: Reboot type to generated service shutdown sequence for. packages: Dict of installed packages. @@ -447,11 +445,11 @@ def generate_shutdown_sequence_file(self, reboot_type: str, packages: Dict[str, order = self.get_shutdown_sequence(reboot_type, packages) with open(os.path.join(ETC_SONIC_PATH, f'{reboot_type}-reboot_order'), 'w') as file: file.write(' '.join(order)) - + def generate_shutdown_sequence_files(self, packages: Dict[str, Package]): - """ Generates shutdown sequence file for fast and warm reboot. + """ Generates shutdown sequence file for fast and warm reboot. (path: /etc/sonic/-reboot_order). - + Args: packages: Dict of installed packages. Returns: @@ -490,7 +488,7 @@ def set_initial_config(self, package): init_cfg = package.manifest['package']['init-cfg'] if not init_cfg: return - + for conn in self.sonic_db.get_connectors(): cfg = conn.get_config() new_cfg = init_cfg.copy() @@ -522,7 +520,7 @@ def remove_config(self, package): def validate_config(self, config): """ Validate configuration through YANG. - + Args: config: Config DB data. Returns: @@ -567,7 +565,7 @@ def uninstall_yang_module(self, package: Package): def install_autogen_cli_all(self, package: Package): """ Install autogenerated CLI plugins for package. - + Args: package: Package Returns: @@ -578,8 +576,8 @@ def install_autogen_cli_all(self, package: Package): self.install_autogen_cli(package, command) def uninstall_autogen_cli_all(self, package: Package): - """ Remove autogenerated CLI plugins for package. - + """ Remove autogenerated CLI plugins for package. + Args: package: Package Returns: @@ -590,8 +588,8 @@ def uninstall_autogen_cli_all(self, package: Package): self.uninstall_autogen_cli(package, command) def install_autogen_cli(self, package: Package, command: str): - """ Install autogenerated CLI plugins for package for particular command. - + """ Install autogenerated CLI plugins for package for particular command. + Args: package: Package. command: Name of command to generate CLI for. @@ -608,8 +606,8 @@ def install_autogen_cli(self, package: Package, command: str): log.debug(f'{command} command line interface autogenerated for {module_name}') def uninstall_autogen_cli(self, package: Package, command: str): - """ Uninstall autogenerated CLI plugins for package for particular command. - + """ Uninstall autogenerated CLI plugins for package for particular command. + Args: package: Package. command: Name of command to remove CLI. diff --git a/sonic_package_manager/service_creator/feature.py b/sonic_package_manager/service_creator/feature.py index e93ddfd44f..eb8e1a0710 100644 --- a/sonic_package_manager/service_creator/feature.py +++ b/sonic_package_manager/service_creator/feature.py @@ -36,7 +36,7 @@ def register(self, state: str = 'disabled', owner: str = 'local'): """ Register feature in CONFIG DBs. - + Args: manifest: Feature's manifest. state: Desired feature admin state. @@ -62,8 +62,8 @@ def register(self, conn.set_entry(FEATURE, name, new_cfg) def deregister(self, name: str): - """ Deregister feature by name. - + """ Deregister feature by name. + Args: name: Name of the feature in CONFIG DB. Returns: @@ -73,15 +73,15 @@ def deregister(self, name: str): db_connetors = self._sonic_db.get_connectors() for conn in db_connetors: conn.set_entry(FEATURE, name, None) - + def update(self, - old_manifest: Manifest, - new_manifest: Manifest): + old_manifest: Manifest, + new_manifest: Manifest): """ Migrate feature configuration. It can be that non-configurable feature entries have to be updated. e.g: "has_timer" for example if the new feature introduces a service timer or name of the service has changed, but user configurable entries are not changed). - + Args: old_manifest: Old feature manifest. new_manifest: New feature manifest. diff --git a/sonic_package_manager/service_creator/sonic_db.py b/sonic_package_manager/service_creator/sonic_db.py index dc81fe0e3f..36d7b4a744 100644 --- a/sonic_package_manager/service_creator/sonic_db.py +++ b/sonic_package_manager/service_creator/sonic_db.py @@ -6,7 +6,8 @@ from swsscommon import swsscommon -from utilities_common.general import load_module_from_source +from config.config_mgmt import sonic_cfggen + from sonic_package_manager.service_creator import ETC_SONIC_PATH from sonic_package_manager.service_creator.utils import in_chroot @@ -15,10 +16,6 @@ INIT_CFG_JSON = os.path.join(ETC_SONIC_PATH, 'init_cfg.json') -# Load sonic-cfggen from source since /usr/local/bin/sonic-cfggen does not have .py extension. -sonic_cfggen = load_module_from_source('sonic_cfggen', '/usr/local/bin/sonic-cfggen') - - class PersistentConfigDbConnector: """ swsscommon.ConfigDBConnector adapter for persistent DBs. """ @@ -66,7 +63,7 @@ def mod_config(self, config): for table_name in config: table_data = config[table_name] if table_data is None: - self._del_table(config, table) + self._del_table(config, table_name) continue for key in table_data: self.mod_entry(table_name, key, table_data[key]) From e2acf73e3ee09e4c70c0bd163b02c77cc102395d Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Wed, 19 May 2021 11:06:32 +0000 Subject: [PATCH 125/173] Fixed 'grouping' parsing Signed-off-by: Vadym Hlushko --- sonic_cli_gen/yang_parser.py | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/sonic_cli_gen/yang_parser.py b/sonic_cli_gen/yang_parser.py index 4a8664b78e..91f65c0da3 100644 --- a/sonic_cli_gen/yang_parser.py +++ b/sonic_cli_gen/yang_parser.py @@ -228,26 +228,30 @@ def on_uses(y_module: OrderedDict, y_uses, conf_mgmt) -> list: """ ret_attrs = list() y_grouping = get_all_grouping(y_module, y_uses, conf_mgmt) + # trim prefixes in order to the next checks + trim_uses_prefixes(y_uses) if y_grouping == []: # not sure if it can happend raise Exception('EMPTY') + # TODO: 'refine' support for group in y_grouping: if isinstance(y_uses, list): for use in y_uses: - if use.get('@name') == group.get('@name'): + if group.get('@name') == use.get('@name'): ret_attrs.extend(get_leafs(group, group.get('@name'))) ret_attrs.extend(get_leaf_lists(group, grouping_name = '')) ret_attrs.extend(get_choices(y_module, group, conf_mgmt)) else: - if y_uses.get('@name') == group.get('@name'): + if group.get('@name') == y_uses.get('@name'): ret_attrs.extend(get_leafs(group, group.get('@name'))) ret_attrs.extend(get_leaf_lists(group, grouping_name = '')) ret_attrs.extend(get_choices(y_module, group, conf_mgmt)) return ret_attrs + def on_choices(y_module: OrderedDict, y_choices, conf_mgmt) -> list: """ Parse a YANG 'choice' entities @@ -378,6 +382,8 @@ def get_uses(y_module: OrderedDict, y_entity: OrderedDict, conf_mgmt) -> list: return [] def get_all_grouping(y_module: OrderedDict, y_uses: OrderedDict, conf_mgmt) -> list: + """ Get all 'grouping' entities that is 'uses' in current YANG + """ # WARNING # TODO add to the design statement that grouping should be defined under the 'module' and NOT in nested containers ret_grouping = list() @@ -458,6 +464,28 @@ def get_import_prefixes(y_uses: OrderedDict) -> list: return ret_prefixes +def trim_uses_prefixes(y_uses) -> list: + """ Trim prefixes from 'uses' YANG entities. + If YANG 'grouping' was imported from another YANG file, it use 'prefix' before 'grouping' name: + { + uses sgrop:endpoint; + } + Where 'sgrop' = 'prefix'; 'endpoint' = 'grouping' name. + + Args: + y_uses - reference to 'uses' + """ + prefixes = get_import_prefixes(y_uses) + + for prefix in prefixes: + if isinstance(y_uses, list): + for use in y_uses: + if prefix in use.get('@name'): + use['@name'] = use.get('@name').split(':')[1] + else: + if prefix in y_uses.get('@name'): + y_uses['@name'] = y_uses.get('@name').split(':')[1] + def get_list_keys(y_list: OrderedDict) -> list: ret_list = list() keys = y_list.get('key').get('@value').split() From 3305faa0e84cce01d5a518981caf51201961e2f6 Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Wed, 19 May 2021 16:52:39 +0000 Subject: [PATCH 126/173] DONE. ALL UT PASSED Signed-off-by: Vadym Hlushko --- sonic_cli_gen/yang_parser.py | 26 +- .../cli_autogen_input/assert_dictionaries.py | 236 ++++++++++++++++++ .../sonic-choice-complex-1.yang | 47 ---- .../sonic-choice-complex.yang | 91 +++++++ tests/cli_autogen_input/sonic-grouping-1.yang | 25 ++ tests/cli_autogen_input/sonic-grouping-2.yang | 25 ++ .../sonic-grouping-complex.yang | 96 +++++++ tests/cli_autogen_input/sonic-grouping.yang | 18 -- tests/cli_autogen_yang_parser_test.py | 40 ++- 9 files changed, 515 insertions(+), 89 deletions(-) delete mode 100644 tests/cli_autogen_input/sonic-choice-complex-1.yang create mode 100644 tests/cli_autogen_input/sonic-choice-complex.yang create mode 100644 tests/cli_autogen_input/sonic-grouping-1.yang create mode 100644 tests/cli_autogen_input/sonic-grouping-2.yang create mode 100644 tests/cli_autogen_input/sonic-grouping-complex.yang delete mode 100644 tests/cli_autogen_input/sonic-grouping.yang diff --git a/sonic_cli_gen/yang_parser.py b/sonic_cli_gen/yang_parser.py index 91f65c0da3..428aa6a23e 100644 --- a/sonic_cli_gen/yang_parser.py +++ b/sonic_cli_gen/yang_parser.py @@ -208,7 +208,7 @@ def on_object_container(y_module: OrderedDict, cont: OrderedDict, conf_mgmt, is_ attrs_list = list() attrs_list.extend(get_leafs(cont, grouping_name = '')) attrs_list.extend(get_leaf_lists(cont, grouping_name = '')) - attrs_list.extend(get_choices(y_module, cont, conf_mgmt)) + attrs_list.extend(get_choices(y_module, cont, conf_mgmt, grouping_name = '')) # TODO: need to test 'grouping' attrs_list.extend(get_uses(y_module, cont, conf_mgmt)) @@ -241,18 +241,18 @@ def on_uses(y_module: OrderedDict, y_uses, conf_mgmt) -> list: for use in y_uses: if group.get('@name') == use.get('@name'): ret_attrs.extend(get_leafs(group, group.get('@name'))) - ret_attrs.extend(get_leaf_lists(group, grouping_name = '')) - ret_attrs.extend(get_choices(y_module, group, conf_mgmt)) + ret_attrs.extend(get_leaf_lists(group, group.get('@name'))) + ret_attrs.extend(get_choices(y_module, group, conf_mgmt, group.get('@name'))) else: if group.get('@name') == y_uses.get('@name'): ret_attrs.extend(get_leafs(group, group.get('@name'))) - ret_attrs.extend(get_leaf_lists(group, grouping_name = '')) - ret_attrs.extend(get_choices(y_module, group, conf_mgmt)) + ret_attrs.extend(get_leaf_lists(group, group.get('@name'))) + ret_attrs.extend(get_choices(y_module, group, conf_mgmt, group.get('@name'))) return ret_attrs -def on_choices(y_module: OrderedDict, y_choices, conf_mgmt) -> list: +def on_choices(y_module: OrderedDict, y_choices, conf_mgmt: ConfigMgmt, grouping_name: str) -> list: """ Parse a YANG 'choice' entities Args: @@ -266,14 +266,14 @@ def on_choices(y_module: OrderedDict, y_choices, conf_mgmt) -> list: # the YANG model can have multiple 'choice' entities inside a 'container' or 'list' if isinstance(y_choices, list): for choice in y_choices: - attrs = on_choice_cases(y_module, choice.get('case'), conf_mgmt) + attrs = on_choice_cases(y_module, choice.get('case'), conf_mgmt, grouping_name) ret_attrs.extend(attrs) else: - ret_attrs = on_choice_cases(y_module, y_choices.get('case'), conf_mgmt) + ret_attrs = on_choice_cases(y_module, y_choices.get('case'), conf_mgmt, grouping_name) return ret_attrs -def on_choice_cases(y_module: OrderedDict, y_cases: list, conf_mgmt) -> list: +def on_choice_cases(y_module: OrderedDict, y_cases: list, conf_mgmt: ConfigMgmt, grouping_name: str) -> list: """ Parse a single YANG 'case' entity from 'choice' entity 'case' element can have inside - 'leaf', 'leaf-list', 'uses' @@ -288,8 +288,8 @@ def on_choice_cases(y_module: OrderedDict, y_cases: list, conf_mgmt) -> list: if isinstance(y_cases, list): for case in y_cases: - ret_attrs.extend(get_leafs(case, grouping_name = '')) - ret_attrs.extend(get_leaf_lists(case, grouping_name = '')) + ret_attrs.extend(get_leafs(case, grouping_name)) + ret_attrs.extend(get_leaf_lists(case, grouping_name)) # TODO: need to deeply test it ret_attrs.extend(get_uses(y_module, case, conf_mgmt)) else: @@ -369,9 +369,9 @@ def get_leaf_lists(y_entity: OrderedDict, grouping_name) -> list: return [] -def get_choices(y_module: OrderedDict, y_entity: OrderedDict, conf_mgmt) -> list: +def get_choices(y_module: OrderedDict, y_entity: OrderedDict, conf_mgmt: ConfigMgmt, grouping_name: str) -> list: if y_entity.get('choice') is not None: - return on_choices(y_module, y_entity.get('choice'), conf_mgmt) + return on_choices(y_module, y_entity.get('choice'), conf_mgmt, grouping_name) return [] diff --git a/tests/cli_autogen_input/assert_dictionaries.py b/tests/cli_autogen_input/assert_dictionaries.py index 751784dae9..263e48366d 100644 --- a/tests/cli_autogen_input/assert_dictionaries.py +++ b/tests/cli_autogen_input/assert_dictionaries.py @@ -386,4 +386,240 @@ ] } ] +} + +choice_complex = { + "tables":[ + { + "description":"TABLE_1 description", + "name":"TABLE_1", + "static-objects":[ + { + "name":"OBJECT_1", + "description":"OBJECT_1 description", + "attrs":[ + { + "name":"LEAF_1", + "description": "", + "is-mandatory": False, + "is-leaf-list": False, + "group": '', + }, + { + "name":"LEAF_LIST_1", + "description": "", + "is-mandatory": False, + "is-leaf-list": True, + "group": '', + }, + { + "name":"GR_1_LEAF_1", + "description": "", + "is-mandatory": False, + "is-leaf-list": False, + "group": "GR_1", + }, + { + "name":"GR_1_LEAF_2", + "description": "", + "is-mandatory": False, + "is-leaf-list": False, + "group": 'GR_1', + }, + { + "name":"LEAF_2", + "description": "", + "is-mandatory": False, + "is-leaf-list": False, + "group": '', + }, + { + "name":"LEAF_3", + "description": "", + "is-mandatory": False, + "is-leaf-list": False, + "group": '', + }, + { + "name":"LEAF_LIST_2", + "description": "", + "is-mandatory": False, + "is-leaf-list": True, + "group": '', + }, + { + "name":"LEAF_LIST_3", + "description": "", + "is-mandatory": False, + "is-leaf-list": True, + "group": '', + }, + { + "name":"GR_5_LEAF_1", + "description": "", + "is-mandatory": False, + "is-leaf-list": False, + "group": 'GR_5', + }, + { + "name":"GR_5_LEAF_2", + "description": "", + "is-mandatory": False, + "is-leaf-list": False, + "group": 'GR_5', + }, + { + "name":"GR_2_LEAF_1", + "description": "", + "is-mandatory": False, + "is-leaf-list": False, + "group": 'GR_2', + }, + { + "name":"GR_2_LEAF_2", + "description": "", + "is-mandatory": False, + "is-leaf-list": False, + "group": 'GR_2', + }, + { + "name":"GR_3_LEAF_1", + "description": "", + "is-mandatory": False, + "is-leaf-list": False, + "group": 'GR_3', + }, + { + "name":"GR_3_LEAF_2", + "description": "", + "is-mandatory": False, + "is-leaf-list": False, + "group": 'GR_3', + }, + ] + } + ] + } + ] +} + +grouping_complex = { + "tables":[ + { + "description":"TABLE_1 description", + "name":"TABLE_1", + "static-objects":[ + { + "name":"OBJECT_1", + "description":"OBJECT_1 description", + "attrs":[ + { + "name":"GR_1_LEAF_1", + "description": "", + "is-mandatory": False, + "is-leaf-list": False, + "group": "GR_1", + }, + { + "name":"GR_1_LEAF_2", + "description": "", + "is-mandatory": False, + "is-leaf-list": False, + "group": 'GR_1', + }, + ] + }, + { + "name":"OBJECT_2", + "description":"OBJECT_2 description", + "attrs":[ + { + "name":"GR_5_LEAF_1", + "description": "", + "is-mandatory": False, + "is-leaf-list": False, + "group": "GR_5", + }, + { + "name":"GR_5_LEAF_LIST_1", + "description": "", + "is-mandatory": False, + "is-leaf-list": True, + "group": "GR_5", + }, + { + "name":"GR_6_LEAF_1", + "description": "", + "is-mandatory": False, + "is-leaf-list": False, + "group": "GR_6", + }, + { + "name":"GR_6_LEAF_2", + "description": "", + "is-mandatory": False, + "is-leaf-list": False, + "group": "GR_6", + }, + { + "name":"GR_6_CASE_1_LEAF_1", + "description": "", + "is-mandatory": False, + "is-leaf-list": False, + "group": "GR_6", + }, + { + "name":"GR_6_CASE_1_LEAF_LIST_1", + "description": "", + "is-mandatory": False, + "is-leaf-list": True, + "group": "GR_6", + }, + { + "name":"GR_6_CASE_2_LEAF_1", + "description": "", + "is-mandatory": False, + "is-leaf-list": False, + "group": "GR_6", + }, + { + "name":"GR_6_CASE_2_LEAF_2", + "description": "", + "is-mandatory": False, + "is-leaf-list": False, + "group": "GR_6", + }, + { + "name":"GR_6_CASE_2_LEAF_LIST_1", + "description": "", + "is-mandatory": False, + "is-leaf-list": True, + "group": "GR_6", + }, + { + "name":"GR_6_CASE_2_LEAF_LIST_2", + "description": "", + "is-mandatory": False, + "is-leaf-list": True, + "group": "GR_6", + }, + { + "name":"GR_4_LEAF_1", + "description": "", + "is-mandatory": False, + "is-leaf-list": False, + "group": "GR_4", + }, + { + "name":"GR_4_LEAF_2", + "description": "", + "is-mandatory": False, + "is-leaf-list": False, + "group": "GR_4", + }, + ] + } + ] + } + ] } \ No newline at end of file diff --git a/tests/cli_autogen_input/sonic-choice-complex-1.yang b/tests/cli_autogen_input/sonic-choice-complex-1.yang deleted file mode 100644 index 4e45717493..0000000000 --- a/tests/cli_autogen_input/sonic-choice-complex-1.yang +++ /dev/null @@ -1,47 +0,0 @@ -module sonic-static-object-complex-1 { - - yang-version 1.1; - - namespace "http://github.com/Azure/static-complex-1"; - prefix static-complex-1; - - import sonic-grouping { - prefix sgrop; - } - - container sonic-static-object-complex-1 { - /* sonic-static-object-complex-1 - top level container */ - - container TABLE_1 { - /* TABLE_1 - table container */ - - description "TABLE_1 description"; - - container OBJECT_1 { - /* OBJECT_1 - object container, it have: - * 1 choice - */ - - description "OBJECT_1 description"; - - choice CHOICE_1 { - case CHOICE_1_CASE_1 { - leaf LEAF_1 { - type uint16; - } - - leaf-list LEAF_LIST_1 { - type string; - } - } - - case CHOICE_1_CASE_2 { - leaf LEAF_2 { - type string; - } - } - } - } - } - } -} \ No newline at end of file diff --git a/tests/cli_autogen_input/sonic-choice-complex.yang b/tests/cli_autogen_input/sonic-choice-complex.yang new file mode 100644 index 0000000000..7d6a66d89f --- /dev/null +++ b/tests/cli_autogen_input/sonic-choice-complex.yang @@ -0,0 +1,91 @@ +module sonic-choice-complex { + + yang-version 1.1; + + namespace "http://github.com/Azure/choice-complex"; + prefix choice-complex; + + import sonic-grouping-1 { + prefix sgroup1; + } + + import sonic-grouping-2 { + prefix sgroup2; + } + + grouping GR_5 { + leaf GR_5_LEAF_1 { + type string; + } + + leaf GR_5_LEAF_2 { + type string; + } + } + + grouping GR_6 { + leaf GR_6_LEAF_1 { + type string; + } + + leaf GR_6_LEAF_2 { + type string; + } + } + + container sonic-choice-complex { + /* sonic-choice-complex - top level container */ + + container TABLE_1 { + /* TABLE_1 - table container */ + + description "TABLE_1 description"; + + container OBJECT_1 { + /* OBJECT_1 - object container, it have + * 1 choice, which have 2 cases. + * first case have: 1 leaf, 1 leaf-list, 1 uses + * second case have: 2 leafs, 2 leaf-lists, 2 uses + */ + + description "OBJECT_1 description"; + + choice CHOICE_1 { + case CHOICE_1_CASE_1 { + leaf LEAF_1 { + type uint16; + } + + leaf-list LEAF_LIST_1 { + type string; + } + + uses sgroup1:GR_1; + } + + case CHOICE_1_CASE_2 { + leaf LEAF_2 { + type string; + } + + leaf LEAF_3 { + type string; + } + + leaf-list LEAF_LIST_2 { + type string; + } + + leaf-list LEAF_LIST_3 { + type string; + } + + uses GR_5; + uses sgroup1:GR_2; + uses sgroup2:GR_3; + } + } + } + } + } +} \ No newline at end of file diff --git a/tests/cli_autogen_input/sonic-grouping-1.yang b/tests/cli_autogen_input/sonic-grouping-1.yang new file mode 100644 index 0000000000..831c3a4ad8 --- /dev/null +++ b/tests/cli_autogen_input/sonic-grouping-1.yang @@ -0,0 +1,25 @@ +module sonic-grouping-1{ + + yang-version 1.1; + + namespace "http://github.com/Azure/s-grouping-1"; + prefix s-grouping-1; + + grouping GR_1 { + leaf GR_1_LEAF_1 { + type string; + } + leaf GR_1_LEAF_2 { + type string; + } + } + + grouping GR_2 { + leaf GR_2_LEAF_1 { + type string; + } + leaf GR_2_LEAF_2 { + type string; + } + } +} \ No newline at end of file diff --git a/tests/cli_autogen_input/sonic-grouping-2.yang b/tests/cli_autogen_input/sonic-grouping-2.yang new file mode 100644 index 0000000000..bfaa13db15 --- /dev/null +++ b/tests/cli_autogen_input/sonic-grouping-2.yang @@ -0,0 +1,25 @@ +module sonic-grouping-2 { + + yang-version 1.1; + + namespace "http://github.com/Azure/s-grouping-2"; + prefix s-grouping-2; + + grouping GR_3 { + leaf GR_3_LEAF_1 { + type string; + } + leaf GR_3_LEAF_2 { + type string; + } + } + + grouping GR_4 { + leaf GR_4_LEAF_1 { + type string; + } + leaf GR_4_LEAF_2 { + type string; + } + } +} \ No newline at end of file diff --git a/tests/cli_autogen_input/sonic-grouping-complex.yang b/tests/cli_autogen_input/sonic-grouping-complex.yang new file mode 100644 index 0000000000..d6ed68563a --- /dev/null +++ b/tests/cli_autogen_input/sonic-grouping-complex.yang @@ -0,0 +1,96 @@ +module sonic-grouping-complex { + + yang-version 1.1; + + namespace "http://github.com/Azure/grouping-complex"; + prefix grouping-complex; + + import sonic-grouping-1 { + prefix sgroup1; + } + + import sonic-grouping-2 { + prefix sgroup2; + } + + grouping GR_5 { + leaf GR_5_LEAF_1 { + type string; + } + + leaf-list GR_5_LEAF_LIST_1 { + type string; + } + } + + grouping GR_6 { + leaf GR_6_LEAF_1 { + type string; + } + + leaf GR_6_LEAF_2 { + type string; + } + + choice GR_6_CHOICE_1 { + case CHOICE_1_CASE_1 { + leaf GR_6_CASE_1_LEAF_1 { + type uint16; + } + + leaf-list GR_6_CASE_1_LEAF_LIST_1 { + type string; + } + } + + case CHOICE_1_CASE_2 { + leaf GR_6_CASE_2_LEAF_1 { + type uint16; + } + + leaf GR_6_CASE_2_LEAF_2 { + type uint16; + } + + leaf-list GR_6_CASE_2_LEAF_LIST_1 { + type string; + } + + leaf-list GR_6_CASE_2_LEAF_LIST_2 { + type string; + } + } + } + } + + container sonic-grouping-complex { + /* sonic-grouping-complex - top level container */ + + container TABLE_1 { + /* TABLE_1 - table container */ + + description "TABLE_1 description"; + + container OBJECT_1 { + /* OBJECT_1 - object container, it have + * 1 choice, which have 2 cases. + * first case have: 1 leaf, 1 leaf-list, 1 uses + * second case have: 2 leafs, 2 leaf-lists, 2 uses + */ + + description "OBJECT_1 description"; + + uses sgroup1:GR_1; + } + + container OBJECT_2 { + + description "OBJECT_2 description"; + + uses GR_5; + uses GR_6; + uses sgroup2:GR_4; + } + } + } +} \ No newline at end of file diff --git a/tests/cli_autogen_input/sonic-grouping.yang b/tests/cli_autogen_input/sonic-grouping.yang deleted file mode 100644 index 59aa0b5948..0000000000 --- a/tests/cli_autogen_input/sonic-grouping.yang +++ /dev/null @@ -1,18 +0,0 @@ -module sonic-grouping{ - - yang-version 1.1; - - namespace "http://github.com/Azure/s-grouping"; - prefix s-grouping; - - grouping target { - leaf t_address { - type inet:ip-address; - description "Target IP address."; - } - leaf t_port { - type inet:port-number; - description "Target port number."; - } - } -} \ No newline at end of file diff --git a/tests/cli_autogen_yang_parser_test.py b/tests/cli_autogen_yang_parser_test.py index 647f9e1907..bb4f634b3b 100644 --- a/tests/cli_autogen_yang_parser_test.py +++ b/tests/cli_autogen_yang_parser_test.py @@ -17,6 +17,9 @@ class TestYangParser: + #create function like 'start' which copy all YANG to location + #create function teardown + def test_1_table_container(self): yang_model_name = 'sonic-1-table-container' template(yang_model_name, assert_dictionaries.one_table_container) @@ -65,17 +68,32 @@ def test_dynamic_object_complex_2(self): yang_model_name = 'sonic-dynamic-object-complex-2' template(yang_model_name, assert_dictionaries.dynamic_object_complex_2) - #def test_choice_complex_1(self): - # """ Test object container with choice that have: 1 leaf, 1 leaf-list, 1 uses - # """ - # yang_model_name = 'sonic-choice-complex' - # grouping_yang = 'sonic-grouping' - # move_yang_model(grouping_yang) - # template(yang_model_name, assert_dictionaries.dynamic_object_complex_2) - - # TODO: UT for choice - # TODO: UT for grouping - + def test_choice_complex(self): + """ Test object container with choice that have complex strucutre: + leafs, leaf-lists, multiple 'uses' from different files + """ + yang_model_name = 'sonic-choice-complex' + grouping_yang_1 = 'sonic-grouping-1' + grouping_yang_2 = 'sonic-grouping-2' + move_yang_model(grouping_yang_1) + move_yang_model(grouping_yang_2) + template(yang_model_name, assert_dictionaries.choice_complex) + remove_yang_model(grouping_yang_1) + remove_yang_model(grouping_yang_2) + + def test_choice_complex(self): + """ Test object container with muplitple 'uses' that using 'grouping' + from different files. The used 'grouping' have a complex strucutre: + leafs, leaf-lists, choices + """ + yang_model_name = 'sonic-grouping-complex' + grouping_yang_1 = 'sonic-grouping-1' + grouping_yang_2 = 'sonic-grouping-2' + move_yang_model(grouping_yang_1) + move_yang_model(grouping_yang_2) + template(yang_model_name, assert_dictionaries.grouping_complex) + remove_yang_model(grouping_yang_1) + remove_yang_model(grouping_yang_2) def template(yang_model_name, correct_dict): config_db_path = os.path.join(test_path, 'cli_autogen_input/config_db.json') From 30e8ac9b9decaa494b2ce403569d0abe602d8dec Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Wed, 19 May 2021 17:53:02 +0000 Subject: [PATCH 127/173] Refactored UT Signed-off-by: Vadym Hlushko --- tests/cli_autogen_yang_parser_test.py | 124 ++++++++++++-------------- 1 file changed, 56 insertions(+), 68 deletions(-) diff --git a/tests/cli_autogen_yang_parser_test.py b/tests/cli_autogen_yang_parser_test.py index bb4f634b3b..c6a81bef50 100644 --- a/tests/cli_autogen_yang_parser_test.py +++ b/tests/cli_autogen_yang_parser_test.py @@ -1,137 +1,125 @@ -import sys import os -import pytest import logging -# debug import pprint from sonic_cli_gen.yang_parser import YangParser from .cli_autogen_input import assert_dictionaries - logger = logging.getLogger(__name__) test_path = os.path.dirname(os.path.abspath(__file__)) yang_models_path = '/usr/local/yang-models' - +test_yang_models = [ + 'sonic-1-table-container', + 'sonic-2-table-containers', + 'sonic-1-object-container', + 'sonic-2-object-containers', + 'sonic-1-list', + 'sonic-2-lists', + 'sonic-static-object-complex-1', + 'sonic-static-object-complex-2', + 'sonic-dynamic-object-complex-1', + 'sonic-dynamic-object-complex-2', + 'sonic-choice-complex', + 'sonic-grouping-complex', + 'sonic-grouping-1', + 'sonic-grouping-2', +] class TestYangParser: - - #create function like 'start' which copy all YANG to location - #create function teardown + @classmethod + def setup_class(cls): + logger.info("SETUP") + os.environ['UTILITIES_UNIT_TESTING'] = "1" + move_yang_models_to_well_know_location() + + @classmethod + def teardown_class(cls): + logger.info("TEARDOWN") + os.environ['UTILITIES_UNIT_TESTING'] = "0" + remove_yang_models_to_well_know_location() def test_1_table_container(self): - yang_model_name = 'sonic-1-table-container' - template(yang_model_name, assert_dictionaries.one_table_container) + template('sonic-1-table-container', assert_dictionaries.one_table_container) def test_2_table_containers(self): - yang_model_name = 'sonic-2-table-containers' - template(yang_model_name, assert_dictionaries.two_table_containers) + template('sonic-2-table-containers', assert_dictionaries.two_table_containers) def test_1_object_container(self): - yang_model_name = 'sonic-1-object-container' - template(yang_model_name, assert_dictionaries.one_object_container) + template('sonic-1-object-container', assert_dictionaries.one_object_container) def test_2_object_containers(self): - yang_model_name = 'sonic-2-object-containers' - template(yang_model_name, assert_dictionaries.two_object_containers) + template('sonic-2-object-containers', assert_dictionaries.two_object_containers) def test_1_list(self): - yang_model_name = 'sonic-1-list' - template(yang_model_name, assert_dictionaries.one_list) + template('sonic-1-list', assert_dictionaries.one_list) def test_2_lists(self): - yang_model_name = 'sonic-2-lists' - template(yang_model_name, assert_dictionaries.two_lists) + template('sonic-2-lists', assert_dictionaries.two_lists) def test_static_object_complex_1(self): """ Test object container with: 1 leaf, 1 leaf-list, 1 choice. """ - yang_model_name = 'sonic-static-object-complex-1' - template(yang_model_name, assert_dictionaries.static_object_complex_1) + template('sonic-static-object-complex-1', assert_dictionaries.static_object_complex_1) def test_static_object_complex_2(self): """ Test object container with: 2 leafs, 2 leaf-lists, 2 choices. """ - yang_model_name = 'sonic-static-object-complex-2' - template(yang_model_name, assert_dictionaries.static_object_complex_2) + template('sonic-static-object-complex-2', assert_dictionaries.static_object_complex_2) def test_dynamic_object_complex_1(self): """ Test object container with: 1 key, 1 leaf, 1 leaf-list, 1 choice. """ - yang_model_name = 'sonic-dynamic-object-complex-1' - template(yang_model_name, assert_dictionaries.dynamic_object_complex_1) + template('sonic-dynamic-object-complex-1', assert_dictionaries.dynamic_object_complex_1) def test_dynamic_object_complex_2(self): """ Test object container with: 2 keys, 2 leafs, 2 leaf-list, 2 choice. """ - yang_model_name = 'sonic-dynamic-object-complex-2' - template(yang_model_name, assert_dictionaries.dynamic_object_complex_2) + template('sonic-dynamic-object-complex-2', assert_dictionaries.dynamic_object_complex_2) def test_choice_complex(self): """ Test object container with choice that have complex strucutre: leafs, leaf-lists, multiple 'uses' from different files """ - yang_model_name = 'sonic-choice-complex' - grouping_yang_1 = 'sonic-grouping-1' - grouping_yang_2 = 'sonic-grouping-2' - move_yang_model(grouping_yang_1) - move_yang_model(grouping_yang_2) - template(yang_model_name, assert_dictionaries.choice_complex) - remove_yang_model(grouping_yang_1) - remove_yang_model(grouping_yang_2) + template('sonic-choice-complex', assert_dictionaries.choice_complex) def test_choice_complex(self): """ Test object container with muplitple 'uses' that using 'grouping' from different files. The used 'grouping' have a complex strucutre: leafs, leaf-lists, choices """ - yang_model_name = 'sonic-grouping-complex' - grouping_yang_1 = 'sonic-grouping-1' - grouping_yang_2 = 'sonic-grouping-2' - move_yang_model(grouping_yang_1) - move_yang_model(grouping_yang_2) - template(yang_model_name, assert_dictionaries.grouping_complex) - remove_yang_model(grouping_yang_1) - remove_yang_model(grouping_yang_2) + template('sonic-grouping-complex', assert_dictionaries.grouping_complex) def template(yang_model_name, correct_dict): config_db_path = os.path.join(test_path, 'cli_autogen_input/config_db.json') - move_yang_model(yang_model_name) parser = YangParser(yang_model_name = yang_model_name, config_db_path = config_db_path, allow_tbl_without_yang = True, debug = False) yang_dict = parser.parse_yang_model() - pretty_log_debug(yang_dict) - assert yang_dict == correct_dict - remove_yang_model(yang_model_name) - -def move_yang_model(yang_model_name): - """ Move provided YANG model to known location for YangParser class - - Args: - yang_model_name: name of provided YANG model +def move_yang_models_to_well_know_location(): + """ Move a test YANG models to known location + in order to be parsed by YangParser class """ - src_path = os.path.join(test_path, 'cli_autogen_input', yang_model_name + '.yang') - cmd = 'sudo cp {} {}'.format(src_path, yang_models_path) - os.system(cmd) - -def remove_yang_model(yang_model_name): - """ Remove YANG model from well known system location - - Args: - yang_model_name: name of provided YANG model + for yang_model in test_yang_models: + src_path = os.path.join(test_path, 'cli_autogen_input', yang_model + '.yang') + cmd = 'sudo cp {} {}'.format(src_path, yang_models_path) + os.system(cmd) + +def remove_yang_models_to_well_know_location(): + """ Remove a test YANG models to known location + in order to be parsed by YangParser class """ - yang_model_path = os.path.join(yang_models_path, yang_model_name + '.yang') - cmd = 'sudo rm {}'.format(yang_model_path) - os.system(cmd) + for yang_model in test_yang_models: + yang_model_path = os.path.join(yang_models_path, yang_model + '.yang') + cmd = 'sudo rm {}'.format(yang_model_path) + os.system(cmd) -# DEBUG function def pretty_log_debug(dictionary): + """ Pretty print of parsed dictionary + """ for line in pprint.pformat(dictionary).split('\n'): logging.debug(line) - From 15aeb8c545f1dd66ee7bead402c37059965cbdac Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Wed, 19 May 2021 18:39:51 +0000 Subject: [PATCH 128/173] Added docstrings, fixed names of variables Signed-off-by: Vadym Hlushko --- sonic_cli_gen/yang_parser.py | 176 ++++++++++++++++++++++++++--------- 1 file changed, 131 insertions(+), 45 deletions(-) diff --git a/sonic_cli_gen/yang_parser.py b/sonic_cli_gen/yang_parser.py index 428aa6a23e..a0bfe3faac 100644 --- a/sonic_cli_gen/yang_parser.py +++ b/sonic_cli_gen/yang_parser.py @@ -117,8 +117,8 @@ def parse_yang_model(self) -> dict: # determine how many (1 or couple) containers a YANG model have after 'top level' container # 'table' container it is a container that goes after 'top level' container if isinstance(self.y_table_containers, list): - for tbl_cont in self.y_table_containers: - y2d_elem = on_table_container(self.y_module, tbl_cont, self.conf_mgmt) + for tbl_container in self.y_table_containers: + y2d_elem = on_table_container(self.y_module, tbl_container, self.conf_mgmt) self.yang_2_dict['tables'].append(y2d_elem) else: y2d_elem = on_table_container(self.y_module, self.y_table_containers, self.conf_mgmt) @@ -128,46 +128,48 @@ def parse_yang_model(self) -> dict: #------------------------------HANDLERS--------------------------------# -def on_table_container(y_module: OrderedDict, tbl_cont: OrderedDict, conf_mgmt) -> dict: +def on_table_container(y_module: OrderedDict, tbl_container: OrderedDict, conf_mgmt: ConfigMgmt) -> dict: """ Parse 'table' container, 'table' container goes after 'top level' container Args: y_module: reference to 'module' - tbl_cont: reference to 'table' container + tbl_container: reference to 'table' container + conf_mgmt: reference to ConfigMgmt class instance, + it have yJson object which contain all parsed YANG models Returns: - dictionary - element for self.yang_2_dict['tables'] + dictionary: element for self.yang_2_dict['tables'] """ y2d_elem = { - 'name': tbl_cont.get('@name'), - 'description': get_description(tbl_cont) + 'name': tbl_container.get('@name'), + 'description': get_description(tbl_container) } # determine if 'table container' have a 'list' entity - if tbl_cont.get('list') is None: + if tbl_container.get('list') is None: y2d_elem['static-objects'] = list() # 'object' container goes after 'table' container # 'object' container have 2 types - list (like sonic-flex_counter.yang) and NOT list (like sonic-device_metadata.yang) - obj_cont = tbl_cont.get('container') - if isinstance(obj_cont, list): - for cont in obj_cont: - static_obj_elem = on_object_container(y_module, cont, conf_mgmt, is_list=False) + obj_container = tbl_container.get('container') + if isinstance(obj_container, list): + for y_container in obj_container: + static_obj_elem = on_object_container(y_module, y_container, conf_mgmt, is_list=False) y2d_elem['static-objects'].append(static_obj_elem) else: - static_obj_elem = on_object_container(y_module, obj_cont, conf_mgmt, is_list=False) + static_obj_elem = on_object_container(y_module, obj_container, conf_mgmt, is_list=False) y2d_elem['static-objects'].append(static_obj_elem) else: y2d_elem['dynamic-objects'] = list() - tbl_cont_lists = tbl_cont.get('list') + tbl_container_lists = tbl_container.get('list') # 'container' can have more than 1 'list' entity - if isinstance(tbl_cont_lists, list): - for _list in tbl_cont_lists: + if isinstance(tbl_container_lists, list): + for _list in tbl_container_lists: dynamic_obj_elem = on_object_container(y_module, _list, conf_mgmt, is_list=True) y2d_elem['dynamic-objects'].append(dynamic_obj_elem) else: - dynamic_obj_elem = on_object_container(y_module, tbl_cont_lists, conf_mgmt, is_list=True) + dynamic_obj_elem = on_object_container(y_module, tbl_container_lists, conf_mgmt, is_list=True) y2d_elem['dynamic-objects'].append(dynamic_obj_elem) # move 'keys' elements from 'attrs' to 'keys' @@ -175,7 +177,7 @@ def on_table_container(y_module: OrderedDict, tbl_cont: OrderedDict, conf_mgmt) return y2d_elem -def on_object_container(y_module: OrderedDict, cont: OrderedDict, conf_mgmt, is_list: bool) -> dict: +def on_object_container(y_module: OrderedDict, y_container: OrderedDict, conf_mgmt, is_list: bool) -> dict: """ Parse a 'object container'. 'Object container' represent OBJECT inside Config DB schema: { @@ -188,52 +190,58 @@ def on_object_container(y_module: OrderedDict, cont: OrderedDict, conf_mgmt, is_ Args: y_module: reference to 'module' - cont: reference to 'object container' + y_container: reference to 'object container' + conf_mgmt: reference to ConfigMgmt class instance, + it have yJson object which contain all parsed YANG models + is_list: boolean flag to determine if container has 'list' Returns: - dictionary - element for y2d_elem['static-objects'] OR y2d_elem['dynamic-objects'] + dictionary: element for y2d_elem['static-objects'] OR y2d_elem['dynamic-objects'] """ - if cont is None: + if y_container is None: return {} obj_elem = { - 'name': cont.get('@name'), - 'description': get_description(cont), + 'name': y_container.get('@name'), + 'description': get_description(y_container), 'attrs': list() } if is_list: - obj_elem['keys'] = get_list_keys(cont) + obj_elem['keys'] = get_list_keys(y_container) attrs_list = list() - attrs_list.extend(get_leafs(cont, grouping_name = '')) - attrs_list.extend(get_leaf_lists(cont, grouping_name = '')) - attrs_list.extend(get_choices(y_module, cont, conf_mgmt, grouping_name = '')) - # TODO: need to test 'grouping' - attrs_list.extend(get_uses(y_module, cont, conf_mgmt)) + # grouping_name is empty because 'grouping' is not used so far + attrs_list.extend(get_leafs(y_container, grouping_name = '')) + attrs_list.extend(get_leaf_lists(y_container, grouping_name = '')) + attrs_list.extend(get_choices(y_module, y_container, conf_mgmt, grouping_name = '')) + attrs_list.extend(get_uses(y_module, y_container, conf_mgmt)) obj_elem['attrs'] = attrs_list return obj_elem -def on_uses(y_module: OrderedDict, y_uses, conf_mgmt) -> list: +def on_uses(y_module: OrderedDict, y_uses, conf_mgmt: ConfigMgmt) -> list: """ Parse a YANG 'uses' entities 'uses' refearing to 'grouping' YANG entity Args: y_module: reference to 'module' y_uses: reference to 'uses' + conf_mgmt: reference to ConfigMgmt class instance, + it have yJson object which contain all parsed YANG model Returns: - dictionary - element for obj_elem['attrs'], 'attrs' contain a parsed 'leafs' + list: element for obj_elem['attrs'], 'attrs' contain a parsed 'leafs' """ + ret_attrs = list() y_grouping = get_all_grouping(y_module, y_uses, conf_mgmt) # trim prefixes in order to the next checks trim_uses_prefixes(y_uses) + # not sure if it can happend if y_grouping == []: - # not sure if it can happend - raise Exception('EMPTY') + raise Exception('Grouping NOT found') # TODO: 'refine' support for group in y_grouping: @@ -251,14 +259,17 @@ def on_uses(y_module: OrderedDict, y_uses, conf_mgmt) -> list: return ret_attrs - def on_choices(y_module: OrderedDict, y_choices, conf_mgmt: ConfigMgmt, grouping_name: str) -> list: """ Parse a YANG 'choice' entities Args: - cont: reference to 'choice' + y_module: reference to 'module' + y_choices: reference to 'choice' element + conf_mgmt: reference to ConfigMgmt class instance, + it have yJson object which contain all parsed YANG model + grouping_name: if YANG entity contain 'uses', this arg represent 'grouping' name Returns: - dictionary - element for obj_elem['attrs'], 'attrs' contain a parsed 'leafs' + list: element for obj_elem['attrs'], 'attrs' contain a parsed 'leafs' """ ret_attrs = list() @@ -280,8 +291,11 @@ def on_choice_cases(y_module: OrderedDict, y_cases: list, conf_mgmt: ConfigMgmt, Args: y_module: reference to 'module' y_cases: reference to 'case' + conf_mgmt: reference to ConfigMgmt class instance, + it have yJson object which contain all parsed YANG model + grouping_name: if YANG entity contain 'uses', this arg represent 'grouping' name Returns: - dictionary - element for obj_elem['attrs'], 'attrs' contain a parsed 'leafs' + list: element for obj_elem['attrs'], 'attrs' contain a parsed 'leafs' """ ret_attrs = list() @@ -302,8 +316,10 @@ def on_leafs(y_leafs, grouping_name, is_leaf_list: bool) -> list: Args: y_leafs: reference to all 'leaf' elements + grouping_name: if YANG entity contain 'uses', this arg represent 'grouping' name + is_leaf_list: boolean to determine if 'leaf-list' was passed in 'y_leafs' arg Returns: - list - list of parsed 'leaf' elements + list: list of parsed 'leaf' elements """ ret_attrs = list() @@ -323,8 +339,10 @@ def on_leaf(leaf: OrderedDict, is_leaf_list: bool, grouping_name: str) -> dict: Args: leaf: reference to a 'leaf' entity + grouping_name: if YANG entity contain 'uses', this arg represent 'grouping' name + is_leaf_list: boolean to determine if 'leaf-list' was passed in 'y_leafs' arg Returns: - dictionary - parsed 'leaf' element + dictionary: parsed 'leaf' element """ attr = { 'name': leaf.get('@name'), @@ -338,6 +356,14 @@ def on_leaf(leaf: OrderedDict, is_leaf_list: bool, grouping_name: str) -> dict: #----------------------GETERS-------------------------# def get_mandatory(y_leaf: OrderedDict) -> bool: + """ Parse 'mandatory' statement for 'leaf' + + Args: + y_leaf: reference to a 'leaf' entity + Returns: + bool: 'leaf' 'mandatory' value + """ + if y_leaf.get('mandatory') is not None: return True @@ -357,34 +383,83 @@ def get_description(y_entity: OrderedDict) -> str: else: return '' -def get_leafs(y_entity: OrderedDict, grouping_name) -> list: +def get_leafs(y_entity: OrderedDict, grouping_name: str) -> list: + """ Check if YANG entity have 'leafs', if so call handler + + Args: + y_entity: reference YANG 'container' or 'list' or 'choice' or 'uses' + grouping_name: if YANG entity contain 'uses', this arg represent 'grouping' name + Returns: + list: list of parsed 'leaf' elements + """ + if y_entity.get('leaf') is not None: return on_leafs(y_entity.get('leaf'), grouping_name, is_leaf_list=False) return [] -def get_leaf_lists(y_entity: OrderedDict, grouping_name) -> list: +def get_leaf_lists(y_entity: OrderedDict, grouping_name: str) -> list: + """ Check if YANG entity have 'leaf-list', if so call handler + + Args: + y_entity: reference YANG 'container' or 'list' or 'choice' or 'uses' + grouping_name: if YANG entity contain 'uses', this arg represent 'grouping' name + Returns: + list: list of parsed 'leaf-list' elements + """ + if y_entity.get('leaf-list') is not None: return on_leafs(y_entity.get('leaf-list'), grouping_name, is_leaf_list=True) return [] def get_choices(y_module: OrderedDict, y_entity: OrderedDict, conf_mgmt: ConfigMgmt, grouping_name: str) -> list: + """ Check if YANG entity have 'choice', if so call handler + + Args: + y_module: reference to 'module' + y_entity: reference YANG 'container' or 'list' or 'choice' or 'uses' + conf_mgmt: reference to ConfigMgmt class instance, + it have yJson object which contain all parsed YANG model + grouping_name: if YANG entity contain 'uses', this arg represent 'grouping' name + Returns: + list: list of parsed elements inside 'choice' + """ + if y_entity.get('choice') is not None: return on_choices(y_module, y_entity.get('choice'), conf_mgmt, grouping_name) return [] -def get_uses(y_module: OrderedDict, y_entity: OrderedDict, conf_mgmt) -> list: +def get_uses(y_module: OrderedDict, y_entity: OrderedDict, conf_mgmt: ConfigMgmt) -> list: + """ Check if YANG entity have 'uses', if so call handler + + Args: + y_module: reference to 'module' + y_entity: reference YANG 'container' or 'list' or 'choice' or 'uses' + conf_mgmt: reference to ConfigMgmt class instance, + it have yJson object which contain all parsed YANG model + Returns: + list: list of parsed elements inside 'grouping' that referenced by 'uses' + """ + if y_entity.get('uses') is not None: return on_uses(y_module, y_entity.get('uses'), conf_mgmt) return [] -def get_all_grouping(y_module: OrderedDict, y_uses: OrderedDict, conf_mgmt) -> list: - """ Get all 'grouping' entities that is 'uses' in current YANG +def get_all_grouping(y_module: OrderedDict, y_uses: OrderedDict, conf_mgmt: ConfigMgmt) -> list: + """ Get all 'grouping' entities that is referenced by 'uses' in current YANG model + + Args: + y_module: reference to 'module' + y_entity: reference to 'uses' + conf_mgmt: reference to ConfigMgmt class instance, + it have yJson object which contain all parsed YANG model + Returns: + list: list of 'grouping' elements """ - # WARNING + # TODO add to the design statement that grouping should be defined under the 'module' and NOT in nested containers ret_grouping = list() prefix_list = get_import_prefixes(y_uses) @@ -475,6 +550,7 @@ def trim_uses_prefixes(y_uses) -> list: Args: y_uses - reference to 'uses' """ + prefixes = get_import_prefixes(y_uses) for prefix in prefixes: @@ -487,6 +563,15 @@ def trim_uses_prefixes(y_uses) -> list: y_uses['@name'] = y_uses.get('@name').split(':')[1] def get_list_keys(y_list: OrderedDict) -> list: + """ Parse YANG 'keys' + If YANG have 'list', inside the list exist 'keys' + + Args: + y_list: reference to 'list' + Returns: + list: parsed keys + """ + ret_list = list() keys = y_list.get('key').get('@value').split() for k in keys: @@ -505,6 +590,7 @@ def change_dyn_obj_struct(dynamic_objects: OrderedDict): Args: dynamic_objects: reference to self.yang_2_dict['dynamic_objects'] """ + for obj in dynamic_objects: for key in obj.get('keys'): for attr in obj.get('attrs'): From ea2c838266170abe0ef4a41a07fc6f04400dff09 Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Wed, 19 May 2021 18:45:47 +0000 Subject: [PATCH 129/173] Added test_grouping_complex to the execution pipeline Signed-off-by: Vadym Hlushko --- tests/cli_autogen_yang_parser_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/cli_autogen_yang_parser_test.py b/tests/cli_autogen_yang_parser_test.py index c6a81bef50..67898503af 100644 --- a/tests/cli_autogen_yang_parser_test.py +++ b/tests/cli_autogen_yang_parser_test.py @@ -83,7 +83,7 @@ def test_choice_complex(self): """ template('sonic-choice-complex', assert_dictionaries.choice_complex) - def test_choice_complex(self): + def test_grouping_complex(self): """ Test object container with muplitple 'uses' that using 'grouping' from different files. The used 'grouping' have a complex strucutre: leafs, leaf-lists, choices From e18c888ea5e6ed574b6e73a4c99c195184fe0423 Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Thu, 20 May 2021 14:28:33 +0000 Subject: [PATCH 130/173] Removed unused test YANG model Signed-off-by: Vadym Hlushko --- .../cli_autogen_input/sonic-test-complex.yang | 115 ------------------ 1 file changed, 115 deletions(-) delete mode 100644 tests/cli_autogen_input/sonic-test-complex.yang diff --git a/tests/cli_autogen_input/sonic-test-complex.yang b/tests/cli_autogen_input/sonic-test-complex.yang deleted file mode 100644 index e4485c9697..0000000000 --- a/tests/cli_autogen_input/sonic-test-complex.yang +++ /dev/null @@ -1,115 +0,0 @@ -module sonic-test-1 { - - yang-version 1.1; - - namespace "http://github.com/Azure/sonic-test-1"; - prefix many-test-1; - - container sonic-test-1 { - /* sonic-test-1 - top level container, it have: - * 2 table containers. Table container - represent Config DB table name. - */ - - container TABLE_1 { - /* TABLE_1 - table container, it have: - * 2 object containers, Object container - represent Confg DB object name. - */ - description "TABLE_1 description"; - - container OBJECT_1 { - /* OBJECT_1 - object container, it have: - * 1 leaf, - * 1 leaf-list - * 1 choice - */ - description "OBJECT_1 description"; - - leaf OBJ_1_LEAF_1 { - description "OBJ_1_LEAF_1 description"; - type string; - } - - leaf-list OBJ_1_LEAF_LIST_1 { - mandatory true; - type string; - } - - choice OBJ_1_CH_1 { - case OBJ_1_CH_1_CASE_1 { - leaf OBJ_1_CH_1_LEAF_1 { - type uint16; - } - } - case OBJ_1_CH_1_CASE_2 { - leaf OBJ_1_CH_1_LEAF_2 { - type string; - } - } - } - } - - container OBJECT_2 { - /* OBJECT_2 - table container, it have: - * 2 leaf, - * 2 leaf-list - * 2 choice - */ - description "OBJECT_2 description"; - - leaf OBJ_2_LEAF_1 { - description "OBJ_2_LEAF_1 description"; - type string; - } - - leaf OBJ_2_LEAF_2 { - mandatory true; - type string; - } - - leaf-list OBJ_2_LEAF_LIST_1 { - description "OBJ_2_LEAF_LIST_1 description"; - mandatory true; - type string; - } - - leaf-list OBJ_2_LEAF_LIST_2 { - type string; - } - - choice OBJ_2_CH_1 { - case OBJ_2_CH_1_CASE_1 { - leaf OBJ_2_CH_1_LEAF_1 { - type uint16; - } - } - case OBJ_2_CH_1_CASE_2 { - leaf OBJ_2_CH_1_LEAF_2 { - type string { - type string; - } - } - } - } - - choice OBJ_2_CH_2 { - case OBJ_2_CH_2_CASE_1 { - leaf OBJ_2_CH_2_LEAF_1 { - type uint16; - } - } - case OBJ_2_CH_2_CASE_2 { - leaf OBJ_2_CH_2_LEAF_2 { - type string { - type string; - } - } - } - } - } - } - - container TABLE_2 { - - } - } -} \ No newline at end of file From 7bd3ec90627378659d2c5516a935037a20d702ee Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Fri, 21 May 2021 13:59:07 +0000 Subject: [PATCH 131/173] Fixed indentation Signed-off-by: Vadym Hlushko --- setup.py | 4 ++-- sonic_cli_gen/__init__.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 9e3f30e3ad..0240125036 100644 --- a/setup.py +++ b/setup.py @@ -57,7 +57,7 @@ 'undebug', 'utilities_common', 'watchdogutil', - 'sonic_cli_gen', + 'sonic_cli_gen', ], package_data={ 'show': ['aliases.ini'], @@ -161,7 +161,7 @@ 'spm = sonic_package_manager.main:cli', 'undebug = undebug.main:cli', 'watchdogutil = watchdogutil.main:watchdogutil', - 'sonic-cli-gen = sonic_cli_gen.main:cli', + 'sonic-cli-gen = sonic_cli_gen.main:cli', ] }, install_requires=[ diff --git a/sonic_cli_gen/__init__.py b/sonic_cli_gen/__init__.py index 7e49cacd56..57da8bc819 100644 --- a/sonic_cli_gen/__init__.py +++ b/sonic_cli_gen/__init__.py @@ -2,4 +2,4 @@ from sonic_cli_gen.generator import CliGenerator -__all__ = ['CliGenerator'] \ No newline at end of file +__all__ = ['CliGenerator'] From d5dea7f8b1b76a74c2329e99eacfe3be2a1b77ce Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Fri, 21 May 2021 15:32:43 +0000 Subject: [PATCH 132/173] Added sonic-cli-gen remove cmd, reworked private initializer functions Signed-off-by: Vadym Hlushko --- sonic_cli_gen/generator.py | 17 +++++++++++---- sonic_cli_gen/main.py | 41 ++++++++++++++++++++++++------------ sonic_cli_gen/yang_parser.py | 41 ++++++++++++++++++------------------ 3 files changed, 62 insertions(+), 37 deletions(-) diff --git a/sonic_cli_gen/generator.py b/sonic_cli_gen/generator.py index d4a9c81ac0..85191c6ce3 100644 --- a/sonic_cli_gen/generator.py +++ b/sonic_cli_gen/generator.py @@ -1,15 +1,15 @@ #!/usr/bin/env python -import jinja2 import os import pkgutil +import jinja2 from sonic_cli_gen.yang_parser import YangParser class CliGenerator: """ SONiC CLI generator. This class provides public API for sonic-cli-gen python library. It can generate config, - show, sonic-clear CLI plugins + show CLI plugins """ def __init__(self): @@ -18,28 +18,37 @@ def __init__(self): self.loader = jinja2.FileSystemLoader(['/usr/share/sonic/templates/sonic-cli-gen/']) self.env = jinja2.Environment(loader=self.loader) + def generate_cli_plugin(self, cli_group, plugin_name): - """ Generate CLI plugin. """ + """ Generate click CLI plugin. """ parser = YangParser(yang_model_name=plugin_name, config_db_path='configDB', allow_tbl_without_yang=True, debug=False) + # yang_dict will be used as an input for templates located in - /usr/share/sonic/templates/sonic-cli-gen/ yang_dict = parser.parse_yang_model() plugin_path = get_cli_plugin_path(cli_group, plugin_name + '_yang.py') template = self.env.get_template(cli_group + '.py.j2') with open(plugin_path, 'w') as plugin_py: plugin_py.write(template.render(yang_dict)) + print('\nAuto-generation successful!\nLocation: {}'.format(plugin_path)) + def remove_cli_plugin(self, cli_group, plugin_name): plugin_path = get_cli_plugin_path(cli_group, plugin_name + '_yang.py') if os.path.exists(plugin_path): os.remove(plugin_path) + print('{} was removed.'.format(plugin_path)) + else: + print('Path {} doest NOT exist!'.format(plugin_path)) + def get_cli_plugin_path(command, plugin_name): pkg_loader = pkgutil.get_loader(f'{command}.plugins.auto') if pkg_loader is None: - raise PackageManagerError(f'Failed to get plugins path for {command} CLI') + raise Exception(f'Failed to get plugins path for {command} CLI') plugins_pkg_path = os.path.dirname(pkg_loader.path) return os.path.join(plugins_pkg_path, plugin_name) + diff --git a/sonic_cli_gen/main.py b/sonic_cli_gen/main.py index 0ee0b030f5..602ff7305d 100644 --- a/sonic_cli_gen/main.py +++ b/sonic_cli_gen/main.py @@ -6,24 +6,39 @@ @click.group() @click.pass_context def cli(ctx): - """ SONiC CLI generator """ - pass + """ SONiC CLI Auto-generator tool.\r + Generate click CLI plugin for 'config' or 'show' CLI groups.\r + CLI plugin will be generated from the YANG model, which should be in:\r\n + /usr/local/yang-models/ \n + Generated CLI plugin will be placed in: \r\n + /usr/local/lib/python3.7/dist-packages//plugins/auto/ + """ + + context = { + 'gen': CliGenerator() + } + ctx.obj = context + @cli.command() -@click.argument('yang_model_name') +@click.argument('cli_group', type = click.Choice(['config', 'show'])) +@click.argument('yang_model_name', type = click.STRING) @click.pass_context -def generate_config(ctx, yang_model_name): - """ Generate CLI plugin (click) for 'config' CLI group. """ - gen = CliGenerator() - gen.generate_cli_plugin(cli_group='config', plugin_name=yang_model_name) +def generate(ctx, cli_group, yang_model_name): + """ Generate click CLI plugin. """ + + ctx.obj['gen'].generate_cli_plugin(cli_group = cli_group, plugin_name = yang_model_name) + @cli.command() -@click.argument('yang_model_name') +@click.argument('cli_group', type = click.Choice(['config', 'show'])) +@click.argument('yang_model_name', type = click.STRING) @click.pass_context -def generate_show(ctx, yang_model_name): - """ Generate CLI plugin (click) for 'show' CLI group. """ - gen = CliGenerator() - gen.generate_cli_plugin(cli_group='show', plugin_name=yang_model_name) +def remove(ctx, cli_group, yang_model_name): + """ Remove generated click CLI plugin from. """ + + ctx.obj['gen'].remove_cli_plugin(cli_group = cli_group, plugin_name = yang_model_name) + if __name__ == '__main__': - cli() \ No newline at end of file + cli() diff --git a/sonic_cli_gen/yang_parser.py b/sonic_cli_gen/yang_parser.py index a0bfe3faac..f8e030f465 100644 --- a/sonic_cli_gen/yang_parser.py +++ b/sonic_cli_gen/yang_parser.py @@ -3,13 +3,14 @@ from collections import OrderedDict from config.config_mgmt import ConfigMgmt +yang_guidelines_link = 'https://github.com/Azure/SONiC/blob/master/doc/mgmt/SONiC_YANG_Model_Guidelines.md' + class YangParser: """ YANG model parser Attributes: yang_model_name: Name of the YANG model file conf_mgmt: Instance of Config Mgmt class to help parse YANG models - idx_yJson: Index of yang_model_file (1 attr) inside conf_mgmt.sy.yJson object y_module: Reference to 'module' entity from YANG model file y_top_level_container: Reference to top level 'container' entity from YANG model file y_table_containers: Reference to 'container' entities from YANG model file @@ -52,7 +53,6 @@ def __init__(self, debug): self.yang_model_name = yang_model_name self.conf_mgmt = None - self.idx_yJson = None self.y_module = None self.y_top_level_container = None self.y_table_containers = None @@ -72,36 +72,36 @@ def _init_yang_module_and_containers(self): self.y_table_containers Raises: - KeyError: if YANG model is invalid or NOT exist + Exception: if YANG model is invalid or NOT exist """ - self._find_index_of_yang_model() + self.y_module = self._find_yang_model_in_yjson_obj() - if self.idx_yJson is None: - raise KeyError('YANG model {} is NOT exist'.format(self.yang_model_name)) - self.y_module = self.conf_mgmt.sy.yJson[self.idx_yJson]['module'] + if self.y_module is None: + raise Exception('The YANG model {} is NOT exist'.format(self.yang_model_name)) if self.y_module.get('container') is None: - raise KeyError('YANG model {} does NOT have "top level container" element \ - Please follow the SONiC YANG model guidelines: \ - https://github.com/Azure/SONiC/blob/master/doc/mgmt/SONiC_YANG_Model_Guidelines.md'.format(self.yang_model_name)) + raise Exception('The YANG model {} does NOT have "top level container" element\ + Please follow the SONiC YANG model guidelines:\n{}'.format(self.yang_model_name, yang_guidelines_link)) self.y_top_level_container = self.y_module.get('container') if self.y_top_level_container.get('container') is None: - raise KeyError('YANG model {} does NOT have "container" element after "top level container" \ - Please follow the SONiC YANG model guidelines: \ - https://github.com/Azure/SONiC/blob/master/doc/mgmt/SONiC_YANG_Model_Guidelines.md'.format(self.yang_model_name)) + raise Exception('The YANG model {} does NOT have "container" element after "top level container"\ + Please follow the SONiC YANG model guidelines:\n{}'.format(self.yang_model_name, yang_guidelines_link)) self.y_table_containers = self.y_top_level_container.get('container') - def _find_index_of_yang_model(self): - """ Find index of provided YANG model inside yJson object, - and save it to self.idx_yJson variable + def _find_yang_model_in_yjson_obj(self) -> OrderedDict: + """ Find provided YANG model inside yJson object, yJson object contain all yang-models parsed from directory - /usr/local/yang-models + + Returns: + reference to yang_model_name """ - for i in range(len(self.conf_mgmt.sy.yJson)): - if (self.conf_mgmt.sy.yJson[i]['module']['@name'] == self.yang_model_name): - self.idx_yJson = i + # TODO: consider to check yJson type + for yang_model in self.conf_mgmt.sy.yJson: + if yang_model.get('module').get('@name') == self.yang_model_name: + return yang_model.get('module') def parse_yang_model(self) -> dict: """ Parse proviced YANG model @@ -597,4 +597,5 @@ def change_dyn_obj_struct(dynamic_objects: OrderedDict): if key.get('name') == attr.get('name'): key['description'] = attr.get('description') obj['attrs'].remove(attr) - break \ No newline at end of file + break + From b0d21719c66c886d88d14b23132b60004158ce44 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Fri, 21 May 2021 19:29:09 +0300 Subject: [PATCH 133/173] [sonic-cli-gen] address review comments Signed-off-by: Stepan Blyschak --- .../templates/sonic-cli-gen/show.py.j2 | 127 ++++++++++++++++-- 1 file changed, 115 insertions(+), 12 deletions(-) diff --git a/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 b/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 index 3859573d72..aad5fc539c 100644 --- a/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 +++ b/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 @@ -1,5 +1,13 @@ {% from "common.j2" import cli_name -%} -""" Autogenerated show CLI plugin """ +""" +Auto-generated show CLI plugin. +{% if source_template is defined %} +Source template: {{ source_template }} +{% endif %} +{% if source_yang_module is defined %} +Source YANG module: {{ source_yang_module }} +{% endif %} +""" import click import tabulate @@ -12,45 +20,131 @@ import utilities_common.cli as clicommon {%- endmacro %} -def print_attr_helper(entry, attr): +def format_attr_value(entry, attr): + """ Helper that formats attribute to be presented in the table output. + + Args: + entry (Dict[str, str]): CONFIG DB entry configuration. + attr (Dict): Attribute metadata. + + Returns: + str: fomatted attribute value. + """ + if attr["is-leaf-list"]: return "\n".join(entry.get(attr["name"], [])) return entry.get(attr["name"], "N/A") -def print_group_helper(entry, attrs): +def format_group_value(entry, attrs): + """ Helper that formats grouped attribute to be presented in the table output. + + Args: + entry (Dict[str, str]): CONFIG DB entry configuration. + attrs (List[Dict]): Attributes metadata that belongs to the same group. + + Returns: + str: fomatted group attributes. + """ + data = [] for attr in attrs: if entry.get(attr["name"]): - data.append((attr["name"] + ":", print_attr_helper(entry, attr))) + data.append((attr["name"] + ":", format_attr_value(entry, attr))) return tabulate.tabulate(data, tablefmt="plain") +{# Generates a python list that represents a row in the table view. +E.g: +Jinja2: +{{ + gen_row("entry", [ + {"name": "leaf1"}, + {"name": "leaf_1"}, + {"name": "leaf_2"}, + {"name": "leaf_3", "group": "group_0"} + ]) +}} +Result: +[ + format_attr_value( + entry, + {'name': 'leaf1'} + ), + format_attr_value( + entry, + {'name': 'leaf_1'} + ), + format_attr_value( + entry, + {'name': 'leaf_2'} + ), + format_group_value( + entry, + [{'name': 'leaf_3', 'group': 'group_0'}] + ), +] +#} {% macro gen_row(entry, attrs) -%} [ {%- for attr in attrs|rejectattr("group", "defined") %} -print_attr_helper({{ entry }}, {{ attr }}), + format_attr_value( + {{ entry }}, + {{ attr }} + ), {%- endfor %} {%- for group, attrs in attrs|selectattr("group", "defined")|groupby("group") %} -print_group_helper({{ entry }}, {{ attrs }}), + format_group_value( + {{ entry }}, + {{ attrs }} + ), {%- endfor %} ] {% endmacro %} +{# Generates a list that represents a header in table view. +E.g: +Jinja2: {{ + gen_header([ + {"name": "key"}, + {"name": "leaf_1"}, + {"name": "leaf_2"}, + {"name": "leaf_3", "group": "group_0"} + ]) + }} + +Result: +[ + "KEY", + "LEAF 1", + "LEAF 2", + "GROUP 0", +] + +#} {% macro gen_header(attrs) -%} [ -{% for attr in attrs|rejectattr("group", "defined") %} -"{{ column_name(attr.name) }}", -{% endfor %} -{% for group, attrs in attrs|selectattr("group", "defined")|groupby("group") %} -"{{ column_name(group) }}", -{% endfor %} +{% for attr in attrs|rejectattr("group", "defined") -%} + "{{ column_name(attr.name) }}", +{% endfor -%} +{% for group, attrs in attrs|selectattr("group", "defined")|groupby("group") -%} + "{{ column_name(group) }}", +{% endfor -%} ] {% endmacro %} {% for table in tables %} {% if "static-objects" in table %} +{# For static objects generate a command group called against table name. +E.g: +@click.group(name="table-name", + cls=clicommon.AliasedGroup) +def TABLE_NAME(): + """ TABLE DESCRIPTION """ + + pass +#} @click.group(name="{{ cli_name(table.name) }}", cls=clicommon.AliasedGroup) def {{ table.name }}(): @@ -59,6 +153,15 @@ def {{ table.name }}(): pass {% for object in table["static-objects"] %} +{# For every object in static table generate a command +in the group to show individual object configuration. +CLI command is named against the object key in DB. +E.g: +@TABLE_NAME.command(name="object-name") +@clicommon.pass_db +def TABLE_NAME_object_name(db): + ... +#} @{{ table.name }}.command(name="{{ cli_name(object.name) }}") @clicommon.pass_db def {{ table.name }}_{{ object.name }}(db): From aa4af1cf05615e794792ee15559d61051d3c9a8f Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Mon, 24 May 2021 07:54:34 +0000 Subject: [PATCH 134/173] Added new lines, fixed some docstrings Signed-off-by: Vadym Hlushko --- sonic_cli_gen/__init__.py | 1 + sonic_cli_gen/generator.py | 9 +++++++-- sonic_cli_gen/main.py | 1 + sonic_cli_gen/yang_parser.py | 26 +++++++++++++++++++++++--- 4 files changed, 32 insertions(+), 5 deletions(-) diff --git a/sonic_cli_gen/__init__.py b/sonic_cli_gen/__init__.py index 57da8bc819..e7e775c0fb 100644 --- a/sonic_cli_gen/__init__.py +++ b/sonic_cli_gen/__init__.py @@ -3,3 +3,4 @@ from sonic_cli_gen.generator import CliGenerator __all__ = ['CliGenerator'] + diff --git a/sonic_cli_gen/generator.py b/sonic_cli_gen/generator.py index 85191c6ce3..48142a9e7b 100644 --- a/sonic_cli_gen/generator.py +++ b/sonic_cli_gen/generator.py @@ -13,14 +13,16 @@ class CliGenerator: """ def __init__(self): - """ Initialize PackageManager. """ + """ Initialize CliGenerator. """ self.loader = jinja2.FileSystemLoader(['/usr/share/sonic/templates/sonic-cli-gen/']) self.env = jinja2.Environment(loader=self.loader) def generate_cli_plugin(self, cli_group, plugin_name): - """ Generate click CLI plugin. """ + """ Generate click CLI plugin and put it to: + /usr/local/lib/python3.7/dist-packages//plugins/auto/ + """ parser = YangParser(yang_model_name=plugin_name, config_db_path='configDB', @@ -36,6 +38,9 @@ def generate_cli_plugin(self, cli_group, plugin_name): def remove_cli_plugin(self, cli_group, plugin_name): + """ Remove CLI plugin from directory: + /usr/local/lib/python3.7/dist-packages//plugins/auto/ + """ plugin_path = get_cli_plugin_path(cli_group, plugin_name + '_yang.py') if os.path.exists(plugin_path): os.remove(plugin_path) diff --git a/sonic_cli_gen/main.py b/sonic_cli_gen/main.py index 602ff7305d..f23249d354 100644 --- a/sonic_cli_gen/main.py +++ b/sonic_cli_gen/main.py @@ -42,3 +42,4 @@ def remove(ctx, cli_group, yang_model_name): if __name__ == '__main__': cli() + diff --git a/sonic_cli_gen/yang_parser.py b/sonic_cli_gen/yang_parser.py index f8e030f465..952a2e60a2 100644 --- a/sonic_cli_gen/yang_parser.py +++ b/sonic_cli_gen/yang_parser.py @@ -29,7 +29,8 @@ class YangParser: 'name': 'value', 'description': 'value', 'is-leaf-list': False, - 'is-mandatory': False + 'is-mandatory': False, + 'group': 'value' } ... ], @@ -126,6 +127,7 @@ def parse_yang_model(self) -> dict: return self.yang_2_dict + #------------------------------HANDLERS--------------------------------# def on_table_container(y_module: OrderedDict, tbl_container: OrderedDict, conf_mgmt: ConfigMgmt) -> dict: @@ -177,6 +179,7 @@ def on_table_container(y_module: OrderedDict, tbl_container: OrderedDict, conf_m return y2d_elem + def on_object_container(y_module: OrderedDict, y_container: OrderedDict, conf_mgmt, is_list: bool) -> dict: """ Parse a 'object container'. 'Object container' represent OBJECT inside Config DB schema: @@ -221,6 +224,7 @@ def on_object_container(y_module: OrderedDict, y_container: OrderedDict, conf_mg return obj_elem + def on_uses(y_module: OrderedDict, y_uses, conf_mgmt: ConfigMgmt) -> list: """ Parse a YANG 'uses' entities 'uses' refearing to 'grouping' YANG entity @@ -259,6 +263,7 @@ def on_uses(y_module: OrderedDict, y_uses, conf_mgmt: ConfigMgmt) -> list: return ret_attrs + def on_choices(y_module: OrderedDict, y_choices, conf_mgmt: ConfigMgmt, grouping_name: str) -> list: """ Parse a YANG 'choice' entities @@ -284,6 +289,7 @@ def on_choices(y_module: OrderedDict, y_choices, conf_mgmt: ConfigMgmt, grouping return ret_attrs + def on_choice_cases(y_module: OrderedDict, y_cases: list, conf_mgmt: ConfigMgmt, grouping_name: str) -> list: """ Parse a single YANG 'case' entity from 'choice' entity 'case' element can have inside - 'leaf', 'leaf-list', 'uses' @@ -304,13 +310,13 @@ def on_choice_cases(y_module: OrderedDict, y_cases: list, conf_mgmt: ConfigMgmt, for case in y_cases: ret_attrs.extend(get_leafs(case, grouping_name)) ret_attrs.extend(get_leaf_lists(case, grouping_name)) - # TODO: need to deeply test it ret_attrs.extend(get_uses(y_module, case, conf_mgmt)) else: raise Exception('It has no sense to using a single "case" element inside "choice" element') return ret_attrs + def on_leafs(y_leafs, grouping_name, is_leaf_list: bool) -> list: """ Parse all the 'leaf' or 'leaf-list' elements @@ -334,6 +340,7 @@ def on_leafs(y_leafs, grouping_name, is_leaf_list: bool) -> list: return ret_attrs + def on_leaf(leaf: OrderedDict, is_leaf_list: bool, grouping_name: str) -> dict: """ Parse a single 'leaf' element @@ -353,6 +360,7 @@ def on_leaf(leaf: OrderedDict, is_leaf_list: bool, grouping_name: str) -> dict: return attr + #----------------------GETERS-------------------------# def get_mandatory(y_leaf: OrderedDict) -> bool: @@ -369,6 +377,7 @@ def get_mandatory(y_leaf: OrderedDict) -> bool: return False + def get_description(y_entity: OrderedDict) -> str: """ Parse 'description' entity from any YANG element @@ -383,6 +392,7 @@ def get_description(y_entity: OrderedDict) -> str: else: return '' + def get_leafs(y_entity: OrderedDict, grouping_name: str) -> list: """ Check if YANG entity have 'leafs', if so call handler @@ -398,6 +408,7 @@ def get_leafs(y_entity: OrderedDict, grouping_name: str) -> list: return [] + def get_leaf_lists(y_entity: OrderedDict, grouping_name: str) -> list: """ Check if YANG entity have 'leaf-list', if so call handler @@ -413,6 +424,7 @@ def get_leaf_lists(y_entity: OrderedDict, grouping_name: str) -> list: return [] + def get_choices(y_module: OrderedDict, y_entity: OrderedDict, conf_mgmt: ConfigMgmt, grouping_name: str) -> list: """ Check if YANG entity have 'choice', if so call handler @@ -431,6 +443,7 @@ def get_choices(y_module: OrderedDict, y_entity: OrderedDict, conf_mgmt: ConfigM return [] + def get_uses(y_module: OrderedDict, y_entity: OrderedDict, conf_mgmt: ConfigMgmt) -> list: """ Check if YANG entity have 'uses', if so call handler @@ -448,6 +461,7 @@ def get_uses(y_module: OrderedDict, y_entity: OrderedDict, conf_mgmt: ConfigMgmt return [] + def get_all_grouping(y_module: OrderedDict, y_uses: OrderedDict, conf_mgmt: ConfigMgmt) -> list: """ Get all 'grouping' entities that is referenced by 'uses' in current YANG model @@ -460,7 +474,6 @@ def get_all_grouping(y_module: OrderedDict, y_uses: OrderedDict, conf_mgmt: Conf list: list of 'grouping' elements """ - # TODO add to the design statement that grouping should be defined under the 'module' and NOT in nested containers ret_grouping = list() prefix_list = get_import_prefixes(y_uses) @@ -487,6 +500,7 @@ def get_all_grouping(y_module: OrderedDict, y_uses: OrderedDict, conf_mgmt: Conf return ret_grouping + def get_grouping_from_another_yang_model(yang_model_name: str, conf_mgmt) -> list: """ Get the YANG 'grouping' entity @@ -498,6 +512,7 @@ def get_grouping_from_another_yang_model(yang_model_name: str, conf_mgmt) -> lis Returns: list - list 'grouping' entities """ + ret_grouping = list() for i in range(len(conf_mgmt.sy.yJson)): @@ -511,6 +526,7 @@ def get_grouping_from_another_yang_model(yang_model_name: str, conf_mgmt) -> lis return ret_grouping + def get_import_prefixes(y_uses: OrderedDict) -> list: """ Parse 'import prefix' of YANG 'uses' entity Example: @@ -525,6 +541,7 @@ def get_import_prefixes(y_uses: OrderedDict) -> list: Returns: list - of parsed prefixes """ + ret_prefixes = list() if isinstance(y_uses, list): @@ -539,6 +556,7 @@ def get_import_prefixes(y_uses: OrderedDict) -> list: return ret_prefixes + def trim_uses_prefixes(y_uses) -> list: """ Trim prefixes from 'uses' YANG entities. If YANG 'grouping' was imported from another YANG file, it use 'prefix' before 'grouping' name: @@ -562,6 +580,7 @@ def trim_uses_prefixes(y_uses) -> list: if prefix in y_uses.get('@name'): y_uses['@name'] = y_uses.get('@name').split(':')[1] + def get_list_keys(y_list: OrderedDict) -> list: """ Parse YANG 'keys' If YANG have 'list', inside the list exist 'keys' @@ -580,6 +599,7 @@ def get_list_keys(y_list: OrderedDict) -> list: return ret_list + def change_dyn_obj_struct(dynamic_objects: OrderedDict): """ Rearrange self.yang_2_dict['dynamic_objects'] structure. If YANG model have a 'list' entity - inside the 'list' it has 'key' entity. From 718a431831589f50ce6ee89f5d3c083fe01956b4 Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Mon, 24 May 2021 08:00:00 +0000 Subject: [PATCH 135/173] removed type from docstring Signed-off-by: Vadym Hlushko --- sonic_cli_gen/yang_parser.py | 42 ++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/sonic_cli_gen/yang_parser.py b/sonic_cli_gen/yang_parser.py index 952a2e60a2..a48b7b707a 100644 --- a/sonic_cli_gen/yang_parser.py +++ b/sonic_cli_gen/yang_parser.py @@ -105,17 +105,17 @@ def _find_yang_model_in_yjson_obj(self) -> OrderedDict: return yang_model.get('module') def parse_yang_model(self) -> dict: - """ Parse proviced YANG model + """ Parse provided YANG model and save output to self.yang_2_dict object Returns: - dictionary - parsed YANG model in dictionary format + parsed YANG model in dictionary format """ self._init_yang_module_and_containers() self.yang_2_dict['tables'] = list() - # determine how many (1 or couple) containers a YANG model have after 'top level' container + # determine how many (1 or more) containers a YANG model have after 'top level' container # 'table' container it is a container that goes after 'top level' container if isinstance(self.y_table_containers, list): for tbl_container in self.y_table_containers: @@ -140,7 +140,7 @@ def on_table_container(y_module: OrderedDict, tbl_container: OrderedDict, conf_m conf_mgmt: reference to ConfigMgmt class instance, it have yJson object which contain all parsed YANG models Returns: - dictionary: element for self.yang_2_dict['tables'] + element for self.yang_2_dict['tables'] """ y2d_elem = { @@ -198,7 +198,7 @@ def on_object_container(y_module: OrderedDict, y_container: OrderedDict, conf_mg it have yJson object which contain all parsed YANG models is_list: boolean flag to determine if container has 'list' Returns: - dictionary: element for y2d_elem['static-objects'] OR y2d_elem['dynamic-objects'] + element for y2d_elem['static-objects'] OR y2d_elem['dynamic-objects'] """ if y_container is None: @@ -235,7 +235,7 @@ def on_uses(y_module: OrderedDict, y_uses, conf_mgmt: ConfigMgmt) -> list: conf_mgmt: reference to ConfigMgmt class instance, it have yJson object which contain all parsed YANG model Returns: - list: element for obj_elem['attrs'], 'attrs' contain a parsed 'leafs' + element for obj_elem['attrs'], 'attrs' contain a parsed 'leafs' """ ret_attrs = list() @@ -274,7 +274,7 @@ def on_choices(y_module: OrderedDict, y_choices, conf_mgmt: ConfigMgmt, grouping it have yJson object which contain all parsed YANG model grouping_name: if YANG entity contain 'uses', this arg represent 'grouping' name Returns: - list: element for obj_elem['attrs'], 'attrs' contain a parsed 'leafs' + element for obj_elem['attrs'], 'attrs' contain a parsed 'leafs' """ ret_attrs = list() @@ -301,7 +301,7 @@ def on_choice_cases(y_module: OrderedDict, y_cases: list, conf_mgmt: ConfigMgmt, it have yJson object which contain all parsed YANG model grouping_name: if YANG entity contain 'uses', this arg represent 'grouping' name Returns: - list: element for obj_elem['attrs'], 'attrs' contain a parsed 'leafs' + element for obj_elem['attrs'], 'attrs' contain a parsed 'leafs' """ ret_attrs = list() @@ -325,7 +325,7 @@ def on_leafs(y_leafs, grouping_name, is_leaf_list: bool) -> list: grouping_name: if YANG entity contain 'uses', this arg represent 'grouping' name is_leaf_list: boolean to determine if 'leaf-list' was passed in 'y_leafs' arg Returns: - list: list of parsed 'leaf' elements + list of parsed 'leaf' elements """ ret_attrs = list() @@ -349,7 +349,7 @@ def on_leaf(leaf: OrderedDict, is_leaf_list: bool, grouping_name: str) -> dict: grouping_name: if YANG entity contain 'uses', this arg represent 'grouping' name is_leaf_list: boolean to determine if 'leaf-list' was passed in 'y_leafs' arg Returns: - dictionary: parsed 'leaf' element + parsed 'leaf' element """ attr = { 'name': leaf.get('@name'), @@ -369,7 +369,7 @@ def get_mandatory(y_leaf: OrderedDict) -> bool: Args: y_leaf: reference to a 'leaf' entity Returns: - bool: 'leaf' 'mandatory' value + 'leaf' 'mandatory' value """ if y_leaf.get('mandatory') is not None: @@ -384,7 +384,7 @@ def get_description(y_entity: OrderedDict) -> str: Args: y_entity: reference to YANG 'container' OR 'list' OR 'leaf' ... Returns: - str - text of the 'description' + text of the 'description' """ if y_entity.get('description') is not None: @@ -400,7 +400,7 @@ def get_leafs(y_entity: OrderedDict, grouping_name: str) -> list: y_entity: reference YANG 'container' or 'list' or 'choice' or 'uses' grouping_name: if YANG entity contain 'uses', this arg represent 'grouping' name Returns: - list: list of parsed 'leaf' elements + list of parsed 'leaf' elements """ if y_entity.get('leaf') is not None: @@ -416,7 +416,7 @@ def get_leaf_lists(y_entity: OrderedDict, grouping_name: str) -> list: y_entity: reference YANG 'container' or 'list' or 'choice' or 'uses' grouping_name: if YANG entity contain 'uses', this arg represent 'grouping' name Returns: - list: list of parsed 'leaf-list' elements + list of parsed 'leaf-list' elements """ if y_entity.get('leaf-list') is not None: @@ -435,7 +435,7 @@ def get_choices(y_module: OrderedDict, y_entity: OrderedDict, conf_mgmt: ConfigM it have yJson object which contain all parsed YANG model grouping_name: if YANG entity contain 'uses', this arg represent 'grouping' name Returns: - list: list of parsed elements inside 'choice' + list of parsed elements inside 'choice' """ if y_entity.get('choice') is not None: @@ -453,7 +453,7 @@ def get_uses(y_module: OrderedDict, y_entity: OrderedDict, conf_mgmt: ConfigMgmt conf_mgmt: reference to ConfigMgmt class instance, it have yJson object which contain all parsed YANG model Returns: - list: list of parsed elements inside 'grouping' that referenced by 'uses' + list of parsed elements inside 'grouping' that referenced by 'uses' """ if y_entity.get('uses') is not None: @@ -471,7 +471,7 @@ def get_all_grouping(y_module: OrderedDict, y_uses: OrderedDict, conf_mgmt: Conf conf_mgmt: reference to ConfigMgmt class instance, it have yJson object which contain all parsed YANG model Returns: - list: list of 'grouping' elements + list of 'grouping' elements """ ret_grouping = list() @@ -510,7 +510,7 @@ def get_grouping_from_another_yang_model(yang_model_name: str, conf_mgmt) -> lis it have yJson object which contain all parsed YANG models Returns: - list - list 'grouping' entities + list 'grouping' entities """ ret_grouping = list() @@ -539,7 +539,7 @@ def get_import_prefixes(y_uses: OrderedDict) -> list: Args: y_uses: refrence to YANG 'uses' Returns: - list - of parsed prefixes + list of parsed prefixes """ ret_prefixes = list() @@ -588,7 +588,7 @@ def get_list_keys(y_list: OrderedDict) -> list: Args: y_list: reference to 'list' Returns: - list: parsed keys + liss of parsed keys """ ret_list = list() @@ -600,7 +600,7 @@ def get_list_keys(y_list: OrderedDict) -> list: return ret_list -def change_dyn_obj_struct(dynamic_objects: OrderedDict): +def change_dyn_obj_struct(dynamic_objects: list): """ Rearrange self.yang_2_dict['dynamic_objects'] structure. If YANG model have a 'list' entity - inside the 'list' it has 'key' entity. 'key' entity it is whitespace-separeted list of 'leafs', those 'leafs' was From 290752358c30c8c9718b90346825d65f923347c1 Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Mon, 24 May 2021 13:52:54 +0000 Subject: [PATCH 136/173] Added list_handler() Signed-off-by: Vadym Hlushko --- sonic_cli_gen/yang_parser.py | 42 +++++++++++++++--------------------- 1 file changed, 17 insertions(+), 25 deletions(-) diff --git a/sonic_cli_gen/yang_parser.py b/sonic_cli_gen/yang_parser.py index a48b7b707a..177fa5478e 100644 --- a/sonic_cli_gen/yang_parser.py +++ b/sonic_cli_gen/yang_parser.py @@ -3,6 +3,8 @@ from collections import OrderedDict from config.config_mgmt import ConfigMgmt +from typing import List, Dict + yang_guidelines_link = 'https://github.com/Azure/SONiC/blob/master/doc/mgmt/SONiC_YANG_Model_Guidelines.md' class YangParser: @@ -117,19 +119,21 @@ def parse_yang_model(self) -> dict: # determine how many (1 or more) containers a YANG model have after 'top level' container # 'table' container it is a container that goes after 'top level' container - if isinstance(self.y_table_containers, list): - for tbl_container in self.y_table_containers: - y2d_elem = on_table_container(self.y_module, tbl_container, self.conf_mgmt) - self.yang_2_dict['tables'].append(y2d_elem) - else: - y2d_elem = on_table_container(self.y_module, self.y_table_containers, self.conf_mgmt) - self.yang_2_dict['tables'].append(y2d_elem) + self.yang_2_dict['tables'] += list_handler(self.y_table_containers, lambda e: on_table_container(self.y_module, e, self.conf_mgmt)) return self.yang_2_dict #------------------------------HANDLERS--------------------------------# +def list_handler(y_entity, callback) -> List[Dict]: + """ Determine if the type of entity is a list, if so - call the callback for every list element """ + if isinstance(y_entity, list): + return [callback(e) for e in y_entity] + else: + return [callback(y_entity)] + + def on_table_container(y_module: OrderedDict, tbl_container: OrderedDict, conf_mgmt: ConfigMgmt) -> dict: """ Parse 'table' container, 'table' container goes after 'top level' container @@ -142,7 +146,6 @@ def on_table_container(y_module: OrderedDict, tbl_container: OrderedDict, conf_m Returns: element for self.yang_2_dict['tables'] """ - y2d_elem = { 'name': tbl_container.get('@name'), 'description': get_description(tbl_container) @@ -153,26 +156,15 @@ def on_table_container(y_module: OrderedDict, tbl_container: OrderedDict, conf_m y2d_elem['static-objects'] = list() # 'object' container goes after 'table' container - # 'object' container have 2 types - list (like sonic-flex_counter.yang) and NOT list (like sonic-device_metadata.yang) obj_container = tbl_container.get('container') - if isinstance(obj_container, list): - for y_container in obj_container: - static_obj_elem = on_object_container(y_module, y_container, conf_mgmt, is_list=False) - y2d_elem['static-objects'].append(static_obj_elem) - else: - static_obj_elem = on_object_container(y_module, obj_container, conf_mgmt, is_list=False) - y2d_elem['static-objects'].append(static_obj_elem) + # 'object' container have 2 types - list (like sonic-flex_counter.yang) and NOT list (like sonic-device_metadata.yang) + y2d_elem['static-objects'] += list_handler(obj_container, lambda e: on_object_container(y_module, e, conf_mgmt, is_list = False)) else: y2d_elem['dynamic-objects'] = list() tbl_container_lists = tbl_container.get('list') + # 'container' can have more than 1 'list' entity - if isinstance(tbl_container_lists, list): - for _list in tbl_container_lists: - dynamic_obj_elem = on_object_container(y_module, _list, conf_mgmt, is_list=True) - y2d_elem['dynamic-objects'].append(dynamic_obj_elem) - else: - dynamic_obj_elem = on_object_container(y_module, tbl_container_lists, conf_mgmt, is_list=True) - y2d_elem['dynamic-objects'].append(dynamic_obj_elem) + y2d_elem['dynamic-objects'] += list_handler(tbl_container_lists, lambda e: on_object_container(y_module, e, conf_mgmt, is_list = True)) # move 'keys' elements from 'attrs' to 'keys' change_dyn_obj_struct(y2d_elem['dynamic-objects']) @@ -566,7 +558,7 @@ def trim_uses_prefixes(y_uses) -> list: Where 'sgrop' = 'prefix'; 'endpoint' = 'grouping' name. Args: - y_uses - reference to 'uses' + y_uses: reference to 'uses' """ prefixes = get_import_prefixes(y_uses) @@ -588,7 +580,7 @@ def get_list_keys(y_list: OrderedDict) -> list: Args: y_list: reference to 'list' Returns: - liss of parsed keys + list of parsed keys """ ret_list = list() From c2b1a32d99caa962e185e2cf52836c85848108eb Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Mon, 24 May 2021 18:37:26 +0300 Subject: [PATCH 137/173] [sonic-cli-gen] fix review comments Signed-off-by: Stepan Blyschak --- .../templates/sonic-cli-gen/config.py.j2 | 149 +++++++++++------- .../templates/sonic-cli-gen/show.py.j2 | 20 ++- 2 files changed, 111 insertions(+), 58 deletions(-) diff --git a/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 b/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 index ac19835cd1..402b7e3dd2 100644 --- a/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 +++ b/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 @@ -1,5 +1,13 @@ {%- from "common.j2" import cli_name -%} -""" Autogenerated config CLI plugin """ +""" +Autogenerated config CLI plugin. +{% if source_template is defined %} +Source template: {{ source_template }} +{% endif %} +{% if source_yang_module is defined %} +Source YANG module: {{ source_yang_module }} +{% endif %} +""" import click import utilities_common.cli as clicommon @@ -11,7 +19,7 @@ from config import config_mgmt sonic_cfggen = general.load_module_from_source('sonic_cfggen', '/usr/local/bin/sonic-cfggen') -def exit_cli(*args, **kwargs): +def exit_with_error(*args, **kwargs): """ Print a message and abort CLI. """ click.secho(*args, **kwargs) @@ -28,20 +36,8 @@ def validate_config_or_raise(cfg): raise Exception('Failed to validate configuration: {}'.format(err)) -def mod_entry_validated(db, table, key, data): - """ Modify existing entry and validate configuration """ - - cfg = db.get_config() - cfg.setdefault(table, {}) - cfg[table].setdefault(key, {}) - cfg[table][key].update(data) - - validate_config_or_raise(cfg) - db.mod_entry(table, key, data) - - def add_entry_validated(db, table, key, data): - """ Add new entry in table and validate configuration""" + """ Add new entry in table and validate configuration """ cfg = db.get_config() cfg.setdefault(table, {}) @@ -54,18 +50,28 @@ def add_entry_validated(db, table, key, data): db.set_entry(table, key, data) -def update_entry_validated(db, table, key, data): - """ Update entry in table and validate configuration""" +def update_entry_validated(db, table, key, data, create_if_not_exists=False): + """ Update entry in table and validate configuration. + If attribute value in data is None, the attribute is deleted. + """ cfg = db.get_config() cfg.setdefault(table, {}) + + if create_if_not_exists: + cfg[table].setdefault(key, {}) + if key not in cfg[table]: raise Exception(f"{key} does not exist") - cfg[table][key].update(data) + for attr, value in data.items(): + if value is None and attr in cfg[table][key]: + cfg[table][key].pop(attr) + else: + cfg[table][key][attr] = value validate_config_or_raise(cfg) - db.mod_entry(table, key, data) + db.set_entry(table, key, cfg[table][key]) def del_entry_validated(db, table, key): @@ -121,33 +127,65 @@ def del_list_entry_validated(db, table, key, attr, data): def clear_list_entry_validated(db, table, key, attr): """ Clear list in object and validate configuration""" - update_entry_validated(db, table, key, {attr: []}) + update_entry_validated(db, table, key, {attr: None}) -{%- macro gen_click_arguments(args) -%} -{%- for arg in args %} +{# Generate click arguments macro +Jinja2 Call: + {{ gen_click_arguments([{"name": "leaf1", "is-leaf-list": False}, + {"name": "leaf2", "is-leaf-list": Talse}) }} +Result: @click.argument( - "{{ cli_name(arg.name) }}", - nargs={% if arg["is-leaf-list"] %}-1{% else %}1{% endif %}, + "leaf1", + nargs=1, + required=True, +) +@click.argument( + "leaf2", + nargs=-1, + required=True, +) +#} +{%- macro gen_click_arguments(attrs) -%} +{%- for attr in attrs %} +@click.argument( + "{{ cli_name(attr.name) }}", + nargs={% if attr["is-leaf-list"] %}-1{% else %}1{% endif %}, required=True, ) {%- endfor %} {%- endmacro %} -{%- macro gen_click_options(opts) -%} -{%- for opt in opts %} + +{# Generate click options macro +Jinja2 Call: + {{ gen_click_arguments([{"name": "leaf1", "is-mandatory": True, "description": "leaf1-desc"}, + {"name": "leaf2", "is-mandatory": False, "description": "leaf2-desc"}) }} +Result: @click.option( - "--{{ cli_name(opt.name) }}", - help="{{ opt.description }}{% if opt['is-mandatory'] %}[mandatory]{% endif %}", + "--leaf1", + help="leaf1-desc [mandatory]", +) +@click.option( + "--leaf2", + help="leaf2-desc", +) +#} +{%- macro gen_click_options(attrs) -%} +{%- for attr in attrs %} +@click.option( + "--{{ cli_name(attr.name) }}", + help="{{ attr.description }}{% if attr['is-mandatory'] %}[mandatory]{% endif %}", ) {%- endfor %} {%- endmacro %} +{# Generate valid python identifier from input names #} {% macro pythonize(attrs) -%} {{ attrs|map(attribute="name")|map("lower")|map("replace", "-", "_")|join(", ") }} {%- endmacro %} -{% macro config_object_list_update(group, table, object, attr) %} +{% macro gen_cfg_obj_list_update(group, table, object, attr) %} {% set list_update_group = group + "_" + attr.name %} @{{ group }}.group(name="{{ cli_name(attr.name) }}", @@ -182,7 +220,7 @@ def {{ list_update_group }}_add( try: add_list_entry_validated(db.cfgdb, table, key, attr, data) except Exception as err: - exit_cli(f"Error: {err}", fg="red") + exit_with_error(f"Error: {err}", fg="red") {# Delete entries from list attribute config CLI generation @@ -210,7 +248,7 @@ def {{ list_update_group }}_delete( try: del_list_entry_validated(db.cfgdb, table, key, attr, data) except Exception as err: - exit_cli(f"Error: {err}", fg="red") + exit_with_error(f"Error: {err}", fg="red") {# Clear entries from list attribute config CLI generation @@ -223,7 +261,7 @@ E.g: @{{ list_update_group }}.command(name="clear") {{ gen_click_arguments(object["keys"]) }} @clicommon.pass_db -def {{ list_update_group }}_delete( +def {{ list_update_group }}_clear( db, {{ pythonize(object["keys"]) }} ): @@ -232,26 +270,25 @@ def {{ list_update_group }}_delete( table = "{{ table.name }}" key = {{ pythonize(object["keys"]) }} attr = "{{ attr.name }}" - data = {{ pythonize([attr]) }} try: clear_list_entry_validated(db.cfgdb, table, key, attr) except Exception as err: - exit_cli(f"Error: {err}", fg="red") + exit_with_error(f"Error: {err}", fg="red") {% endmacro %} -{% macro config_object_list_update_all(group, table, object) %} +{% macro gen_cfg_obj_list_update_all(group, table, object) %} {% for attr in object.attrs %} {% if attr["is-leaf-list"] %} -{{ config_object_list_update(group, table, object, attr) }} +{{ gen_cfg_obj_list_update(group, table, object, attr) }} {% endif %} {% endfor %} {% endmacro %} -{% macro config_static_object_attr(table, object, attr) %} +{% macro gen_cfg_static_obj_attr(table, object, attr) %} @{{ table.name }}_{{ object.name }}.command(name="{{ cli_name(attr.name) }}") {{ gen_click_arguments([attr]) }} @clicommon.pass_db @@ -264,9 +301,9 @@ def {{ table.name }}_{{ object.name }}_{{ attr.name }}(db, {{ pythonize([attr]) "{{ attr.name }}": {{ pythonize([attr]) }}, } try: - mod_entry_validated(db.cfgdb, table, key, data) + update_entry_validated(db.cfgdb, table, key, data, create_if_not_exists=True) except Exception as err: - exit_cli(f"Error: {err}", fg="red") + exit_with_error(f"Error: {err}", fg="red") {% endmacro %} @@ -275,7 +312,7 @@ E.g: @TABLE.group(name="object") def TABLE_object(db): #} -{% macro config_static_object(table, object) %} +{% macro gen_cfg_static_obj(table, object) %} @{{ table.name }}.group(name="{{ cli_name(object.name) }}", cls=clicommon.AliasedGroup) @clicommon.pass_db @@ -290,10 +327,10 @@ E.g: def TABLE_object_attribute(db, attribute): #} {% for attr in object.attrs %} -{{ config_static_object_attr(table, object, attr) }} +{{ gen_cfg_static_obj_attr(table, object, attr) }} {% endfor %} -{{ config_object_list_update_all(table.name + "_" + object.name, table, object) }} +{{ gen_cfg_obj_list_update_all(table.name + "_" + object.name, table, object) }} {% endmacro %} {# Dynamic objects config CLI generation #} @@ -308,7 +345,7 @@ E.g: @click.option("--attr3") def TABLE_TABLE_LIST_add(db, key1, key2, attr1, attr2, attr3): #} -{% macro config_dynamic_object_add(group, table, object) %} +{% macro gen_cfg_dyn_obj_add(group, table, object) %} @{{ group }}.command(name="add") {{ gen_click_arguments(object["keys"]) }} {{ gen_click_options(object.attrs) }} @@ -331,7 +368,7 @@ def {{ group }}_add(db, {{ pythonize(object["keys"] + object.attrs) }}): try: add_entry_validated(db.cfgdb, table, key, data) except Exception as err: - exit_cli(f"Error: {err}", fg="red") + exit_with_error(f"Error: {err}", fg="red") {% endmacro %} {# Dynamic objects update command @@ -344,7 +381,7 @@ E.g: @click.option("--attr3") def TABLE_TABLE_LIST_update(db, key1, key2, attr1, attr2, attr3): #} -{% macro config_dynamic_object_update(group, table, object) %} +{% macro gen_cfg_dyn_obj_update(group, table, object) %} @{{ group }}.command(name="update") {{ gen_click_arguments(object["keys"]) }} {{ gen_click_options(object.attrs) }} @@ -367,7 +404,7 @@ def {{ group }}_update(db, {{ pythonize(object["keys"] + object.attrs) }}): try: update_entry_validated(db.cfgdb, table, key, data) except Exception as err: - exit_cli(f"Error: {err}", fg="red") + exit_with_error(f"Error: {err}", fg="red") {% endmacro %} {# Dynamic objects delete command @@ -377,7 +414,7 @@ E.g: @click.argument("key2") def TABLE_TABLE_LIST_delete(db, key1, key2): #} -{% macro config_dynamic_object_delete(group, table, object) %} +{% macro gen_cfg_dyn_obj_delete(group, table, object) %} @{{ group }}.command(name="delete") {{ gen_click_arguments(object["keys"]) }} @clicommon.pass_db @@ -389,11 +426,11 @@ def {{ group }}_delete(db, {{ pythonize(object["keys"]) }}): try: del_entry_validated(db.cfgdb, table, key) except Exception as err: - exit_cli(f"Error: {err}", fg="red") + exit_with_error(f"Error: {err}", fg="red") {% endmacro %} -{% macro config_dynamic_object(table, object) %} -{# Generate another nesting group in case table holds two types of objects #} +{% macro gen_cfg_dyn_obj(table, object) %} +{# Generate another nested group in case table holds two types of objects #} {% if table["dynamic-objects"]|length > 1 %} {% set group = table.name + "_" + object.name %} @{{ table.name }}.group(name="{{ cli_name(object.name) }}", @@ -406,10 +443,10 @@ def {{ group }}(): {% set group = table.name %} {% endif %} -{{ config_dynamic_object_add(group, table, object) }} -{{ config_dynamic_object_update(group, table, object) }} -{{ config_dynamic_object_delete(group, table, object) }} -{{ config_object_list_update_all(group, table, object) }} +{{ gen_cfg_dyn_obj_add(group, table, object) }} +{{ gen_cfg_dyn_obj_update(group, table, object) }} +{{ gen_cfg_dyn_obj_delete(group, table, object) }} +{{ gen_cfg_obj_list_update_all(group, table, object) }} {% endmacro %} @@ -423,13 +460,13 @@ def {{ table.name }}(): {% if "static-objects" in table %} {% for object in table["static-objects"] %} -{{ config_static_object(table, object) }} +{{ gen_cfg_static_obj(table, object) }} {% endfor %} {% endif %} {% if "dynamic-objects" in table %} {% for object in table["dynamic-objects"] %} -{{ config_dynamic_object(table, object) }} +{{ gen_cfg_dyn_obj(table, object) }} {% endfor %} {% endif %} diff --git a/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 b/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 index aad5fc539c..6ee27f2013 100644 --- a/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 +++ b/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 @@ -94,10 +94,19 @@ Result: ), {%- endfor %} {%- for group, attrs in attrs|selectattr("group", "defined")|groupby("group") %} +{%- if group == "" %} +{%- for attr in attrs %} + format_attr_value( + {{ entry }}, + {{ attr }} + ), +{%- endfor %} +{%- else %} format_group_value( {{ entry }}, {{ attrs }} ), +{%- endif %} {%- endfor %} ] {% endmacro %} @@ -128,7 +137,13 @@ Result: "{{ column_name(attr.name) }}", {% endfor -%} {% for group, attrs in attrs|selectattr("group", "defined")|groupby("group") -%} +{%- if group == "" %} +{% for attr in attrs -%} + "{{ column_name(attr.name) }}", +{% endfor -%} +{%- else %} "{{ column_name(group) }}", +{%- endif %} {% endfor -%} ] {% endmacro %} @@ -148,7 +163,7 @@ def TABLE_NAME(): @click.group(name="{{ cli_name(table.name) }}", cls=clicommon.AliasedGroup) def {{ table.name }}(): - """ {{ table.description }}""" + """ {{ table.description }} """ pass @@ -196,12 +211,13 @@ def {{ table.name }}(): {% set name = table.name %} {% endif %} +{# Generate an implementation to display table. #} @{{ group }}.group(name="{{ cli_name(name) }}", cls=clicommon.AliasedGroup, invoke_without_command=True) @clicommon.pass_db def {{ name }}(db): - """ {{ object.description }} """ + """ {{ object.description }} [Callable command group] """ header = {{ gen_header(object["keys"] + object.attrs) }} body = [] From a005d6d6120370833259806169c3e0babad665c1 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Fri, 21 May 2021 19:29:09 +0300 Subject: [PATCH 138/173] [sonic-cli-gen] address review comments Signed-off-by: Stepan Blyschak --- .../templates/sonic-cli-gen/show.py.j2 | 127 ++++++++++++++++-- 1 file changed, 115 insertions(+), 12 deletions(-) diff --git a/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 b/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 index 3859573d72..aad5fc539c 100644 --- a/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 +++ b/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 @@ -1,5 +1,13 @@ {% from "common.j2" import cli_name -%} -""" Autogenerated show CLI plugin """ +""" +Auto-generated show CLI plugin. +{% if source_template is defined %} +Source template: {{ source_template }} +{% endif %} +{% if source_yang_module is defined %} +Source YANG module: {{ source_yang_module }} +{% endif %} +""" import click import tabulate @@ -12,45 +20,131 @@ import utilities_common.cli as clicommon {%- endmacro %} -def print_attr_helper(entry, attr): +def format_attr_value(entry, attr): + """ Helper that formats attribute to be presented in the table output. + + Args: + entry (Dict[str, str]): CONFIG DB entry configuration. + attr (Dict): Attribute metadata. + + Returns: + str: fomatted attribute value. + """ + if attr["is-leaf-list"]: return "\n".join(entry.get(attr["name"], [])) return entry.get(attr["name"], "N/A") -def print_group_helper(entry, attrs): +def format_group_value(entry, attrs): + """ Helper that formats grouped attribute to be presented in the table output. + + Args: + entry (Dict[str, str]): CONFIG DB entry configuration. + attrs (List[Dict]): Attributes metadata that belongs to the same group. + + Returns: + str: fomatted group attributes. + """ + data = [] for attr in attrs: if entry.get(attr["name"]): - data.append((attr["name"] + ":", print_attr_helper(entry, attr))) + data.append((attr["name"] + ":", format_attr_value(entry, attr))) return tabulate.tabulate(data, tablefmt="plain") +{# Generates a python list that represents a row in the table view. +E.g: +Jinja2: +{{ + gen_row("entry", [ + {"name": "leaf1"}, + {"name": "leaf_1"}, + {"name": "leaf_2"}, + {"name": "leaf_3", "group": "group_0"} + ]) +}} +Result: +[ + format_attr_value( + entry, + {'name': 'leaf1'} + ), + format_attr_value( + entry, + {'name': 'leaf_1'} + ), + format_attr_value( + entry, + {'name': 'leaf_2'} + ), + format_group_value( + entry, + [{'name': 'leaf_3', 'group': 'group_0'}] + ), +] +#} {% macro gen_row(entry, attrs) -%} [ {%- for attr in attrs|rejectattr("group", "defined") %} -print_attr_helper({{ entry }}, {{ attr }}), + format_attr_value( + {{ entry }}, + {{ attr }} + ), {%- endfor %} {%- for group, attrs in attrs|selectattr("group", "defined")|groupby("group") %} -print_group_helper({{ entry }}, {{ attrs }}), + format_group_value( + {{ entry }}, + {{ attrs }} + ), {%- endfor %} ] {% endmacro %} +{# Generates a list that represents a header in table view. +E.g: +Jinja2: {{ + gen_header([ + {"name": "key"}, + {"name": "leaf_1"}, + {"name": "leaf_2"}, + {"name": "leaf_3", "group": "group_0"} + ]) + }} + +Result: +[ + "KEY", + "LEAF 1", + "LEAF 2", + "GROUP 0", +] + +#} {% macro gen_header(attrs) -%} [ -{% for attr in attrs|rejectattr("group", "defined") %} -"{{ column_name(attr.name) }}", -{% endfor %} -{% for group, attrs in attrs|selectattr("group", "defined")|groupby("group") %} -"{{ column_name(group) }}", -{% endfor %} +{% for attr in attrs|rejectattr("group", "defined") -%} + "{{ column_name(attr.name) }}", +{% endfor -%} +{% for group, attrs in attrs|selectattr("group", "defined")|groupby("group") -%} + "{{ column_name(group) }}", +{% endfor -%} ] {% endmacro %} {% for table in tables %} {% if "static-objects" in table %} +{# For static objects generate a command group called against table name. +E.g: +@click.group(name="table-name", + cls=clicommon.AliasedGroup) +def TABLE_NAME(): + """ TABLE DESCRIPTION """ + + pass +#} @click.group(name="{{ cli_name(table.name) }}", cls=clicommon.AliasedGroup) def {{ table.name }}(): @@ -59,6 +153,15 @@ def {{ table.name }}(): pass {% for object in table["static-objects"] %} +{# For every object in static table generate a command +in the group to show individual object configuration. +CLI command is named against the object key in DB. +E.g: +@TABLE_NAME.command(name="object-name") +@clicommon.pass_db +def TABLE_NAME_object_name(db): + ... +#} @{{ table.name }}.command(name="{{ cli_name(object.name) }}") @clicommon.pass_db def {{ table.name }}_{{ object.name }}(db): From aa00b616f7851caf3cba8f861d7201d9bd52724f Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Mon, 24 May 2021 18:37:26 +0300 Subject: [PATCH 139/173] [sonic-cli-gen] fix review comments Signed-off-by: Stepan Blyschak --- .../templates/sonic-cli-gen/config.py.j2 | 149 +++++++++++------- .../templates/sonic-cli-gen/show.py.j2 | 20 ++- 2 files changed, 111 insertions(+), 58 deletions(-) diff --git a/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 b/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 index ac19835cd1..402b7e3dd2 100644 --- a/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 +++ b/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 @@ -1,5 +1,13 @@ {%- from "common.j2" import cli_name -%} -""" Autogenerated config CLI plugin """ +""" +Autogenerated config CLI plugin. +{% if source_template is defined %} +Source template: {{ source_template }} +{% endif %} +{% if source_yang_module is defined %} +Source YANG module: {{ source_yang_module }} +{% endif %} +""" import click import utilities_common.cli as clicommon @@ -11,7 +19,7 @@ from config import config_mgmt sonic_cfggen = general.load_module_from_source('sonic_cfggen', '/usr/local/bin/sonic-cfggen') -def exit_cli(*args, **kwargs): +def exit_with_error(*args, **kwargs): """ Print a message and abort CLI. """ click.secho(*args, **kwargs) @@ -28,20 +36,8 @@ def validate_config_or_raise(cfg): raise Exception('Failed to validate configuration: {}'.format(err)) -def mod_entry_validated(db, table, key, data): - """ Modify existing entry and validate configuration """ - - cfg = db.get_config() - cfg.setdefault(table, {}) - cfg[table].setdefault(key, {}) - cfg[table][key].update(data) - - validate_config_or_raise(cfg) - db.mod_entry(table, key, data) - - def add_entry_validated(db, table, key, data): - """ Add new entry in table and validate configuration""" + """ Add new entry in table and validate configuration """ cfg = db.get_config() cfg.setdefault(table, {}) @@ -54,18 +50,28 @@ def add_entry_validated(db, table, key, data): db.set_entry(table, key, data) -def update_entry_validated(db, table, key, data): - """ Update entry in table and validate configuration""" +def update_entry_validated(db, table, key, data, create_if_not_exists=False): + """ Update entry in table and validate configuration. + If attribute value in data is None, the attribute is deleted. + """ cfg = db.get_config() cfg.setdefault(table, {}) + + if create_if_not_exists: + cfg[table].setdefault(key, {}) + if key not in cfg[table]: raise Exception(f"{key} does not exist") - cfg[table][key].update(data) + for attr, value in data.items(): + if value is None and attr in cfg[table][key]: + cfg[table][key].pop(attr) + else: + cfg[table][key][attr] = value validate_config_or_raise(cfg) - db.mod_entry(table, key, data) + db.set_entry(table, key, cfg[table][key]) def del_entry_validated(db, table, key): @@ -121,33 +127,65 @@ def del_list_entry_validated(db, table, key, attr, data): def clear_list_entry_validated(db, table, key, attr): """ Clear list in object and validate configuration""" - update_entry_validated(db, table, key, {attr: []}) + update_entry_validated(db, table, key, {attr: None}) -{%- macro gen_click_arguments(args) -%} -{%- for arg in args %} +{# Generate click arguments macro +Jinja2 Call: + {{ gen_click_arguments([{"name": "leaf1", "is-leaf-list": False}, + {"name": "leaf2", "is-leaf-list": Talse}) }} +Result: @click.argument( - "{{ cli_name(arg.name) }}", - nargs={% if arg["is-leaf-list"] %}-1{% else %}1{% endif %}, + "leaf1", + nargs=1, + required=True, +) +@click.argument( + "leaf2", + nargs=-1, + required=True, +) +#} +{%- macro gen_click_arguments(attrs) -%} +{%- for attr in attrs %} +@click.argument( + "{{ cli_name(attr.name) }}", + nargs={% if attr["is-leaf-list"] %}-1{% else %}1{% endif %}, required=True, ) {%- endfor %} {%- endmacro %} -{%- macro gen_click_options(opts) -%} -{%- for opt in opts %} + +{# Generate click options macro +Jinja2 Call: + {{ gen_click_arguments([{"name": "leaf1", "is-mandatory": True, "description": "leaf1-desc"}, + {"name": "leaf2", "is-mandatory": False, "description": "leaf2-desc"}) }} +Result: @click.option( - "--{{ cli_name(opt.name) }}", - help="{{ opt.description }}{% if opt['is-mandatory'] %}[mandatory]{% endif %}", + "--leaf1", + help="leaf1-desc [mandatory]", +) +@click.option( + "--leaf2", + help="leaf2-desc", +) +#} +{%- macro gen_click_options(attrs) -%} +{%- for attr in attrs %} +@click.option( + "--{{ cli_name(attr.name) }}", + help="{{ attr.description }}{% if attr['is-mandatory'] %}[mandatory]{% endif %}", ) {%- endfor %} {%- endmacro %} +{# Generate valid python identifier from input names #} {% macro pythonize(attrs) -%} {{ attrs|map(attribute="name")|map("lower")|map("replace", "-", "_")|join(", ") }} {%- endmacro %} -{% macro config_object_list_update(group, table, object, attr) %} +{% macro gen_cfg_obj_list_update(group, table, object, attr) %} {% set list_update_group = group + "_" + attr.name %} @{{ group }}.group(name="{{ cli_name(attr.name) }}", @@ -182,7 +220,7 @@ def {{ list_update_group }}_add( try: add_list_entry_validated(db.cfgdb, table, key, attr, data) except Exception as err: - exit_cli(f"Error: {err}", fg="red") + exit_with_error(f"Error: {err}", fg="red") {# Delete entries from list attribute config CLI generation @@ -210,7 +248,7 @@ def {{ list_update_group }}_delete( try: del_list_entry_validated(db.cfgdb, table, key, attr, data) except Exception as err: - exit_cli(f"Error: {err}", fg="red") + exit_with_error(f"Error: {err}", fg="red") {# Clear entries from list attribute config CLI generation @@ -223,7 +261,7 @@ E.g: @{{ list_update_group }}.command(name="clear") {{ gen_click_arguments(object["keys"]) }} @clicommon.pass_db -def {{ list_update_group }}_delete( +def {{ list_update_group }}_clear( db, {{ pythonize(object["keys"]) }} ): @@ -232,26 +270,25 @@ def {{ list_update_group }}_delete( table = "{{ table.name }}" key = {{ pythonize(object["keys"]) }} attr = "{{ attr.name }}" - data = {{ pythonize([attr]) }} try: clear_list_entry_validated(db.cfgdb, table, key, attr) except Exception as err: - exit_cli(f"Error: {err}", fg="red") + exit_with_error(f"Error: {err}", fg="red") {% endmacro %} -{% macro config_object_list_update_all(group, table, object) %} +{% macro gen_cfg_obj_list_update_all(group, table, object) %} {% for attr in object.attrs %} {% if attr["is-leaf-list"] %} -{{ config_object_list_update(group, table, object, attr) }} +{{ gen_cfg_obj_list_update(group, table, object, attr) }} {% endif %} {% endfor %} {% endmacro %} -{% macro config_static_object_attr(table, object, attr) %} +{% macro gen_cfg_static_obj_attr(table, object, attr) %} @{{ table.name }}_{{ object.name }}.command(name="{{ cli_name(attr.name) }}") {{ gen_click_arguments([attr]) }} @clicommon.pass_db @@ -264,9 +301,9 @@ def {{ table.name }}_{{ object.name }}_{{ attr.name }}(db, {{ pythonize([attr]) "{{ attr.name }}": {{ pythonize([attr]) }}, } try: - mod_entry_validated(db.cfgdb, table, key, data) + update_entry_validated(db.cfgdb, table, key, data, create_if_not_exists=True) except Exception as err: - exit_cli(f"Error: {err}", fg="red") + exit_with_error(f"Error: {err}", fg="red") {% endmacro %} @@ -275,7 +312,7 @@ E.g: @TABLE.group(name="object") def TABLE_object(db): #} -{% macro config_static_object(table, object) %} +{% macro gen_cfg_static_obj(table, object) %} @{{ table.name }}.group(name="{{ cli_name(object.name) }}", cls=clicommon.AliasedGroup) @clicommon.pass_db @@ -290,10 +327,10 @@ E.g: def TABLE_object_attribute(db, attribute): #} {% for attr in object.attrs %} -{{ config_static_object_attr(table, object, attr) }} +{{ gen_cfg_static_obj_attr(table, object, attr) }} {% endfor %} -{{ config_object_list_update_all(table.name + "_" + object.name, table, object) }} +{{ gen_cfg_obj_list_update_all(table.name + "_" + object.name, table, object) }} {% endmacro %} {# Dynamic objects config CLI generation #} @@ -308,7 +345,7 @@ E.g: @click.option("--attr3") def TABLE_TABLE_LIST_add(db, key1, key2, attr1, attr2, attr3): #} -{% macro config_dynamic_object_add(group, table, object) %} +{% macro gen_cfg_dyn_obj_add(group, table, object) %} @{{ group }}.command(name="add") {{ gen_click_arguments(object["keys"]) }} {{ gen_click_options(object.attrs) }} @@ -331,7 +368,7 @@ def {{ group }}_add(db, {{ pythonize(object["keys"] + object.attrs) }}): try: add_entry_validated(db.cfgdb, table, key, data) except Exception as err: - exit_cli(f"Error: {err}", fg="red") + exit_with_error(f"Error: {err}", fg="red") {% endmacro %} {# Dynamic objects update command @@ -344,7 +381,7 @@ E.g: @click.option("--attr3") def TABLE_TABLE_LIST_update(db, key1, key2, attr1, attr2, attr3): #} -{% macro config_dynamic_object_update(group, table, object) %} +{% macro gen_cfg_dyn_obj_update(group, table, object) %} @{{ group }}.command(name="update") {{ gen_click_arguments(object["keys"]) }} {{ gen_click_options(object.attrs) }} @@ -367,7 +404,7 @@ def {{ group }}_update(db, {{ pythonize(object["keys"] + object.attrs) }}): try: update_entry_validated(db.cfgdb, table, key, data) except Exception as err: - exit_cli(f"Error: {err}", fg="red") + exit_with_error(f"Error: {err}", fg="red") {% endmacro %} {# Dynamic objects delete command @@ -377,7 +414,7 @@ E.g: @click.argument("key2") def TABLE_TABLE_LIST_delete(db, key1, key2): #} -{% macro config_dynamic_object_delete(group, table, object) %} +{% macro gen_cfg_dyn_obj_delete(group, table, object) %} @{{ group }}.command(name="delete") {{ gen_click_arguments(object["keys"]) }} @clicommon.pass_db @@ -389,11 +426,11 @@ def {{ group }}_delete(db, {{ pythonize(object["keys"]) }}): try: del_entry_validated(db.cfgdb, table, key) except Exception as err: - exit_cli(f"Error: {err}", fg="red") + exit_with_error(f"Error: {err}", fg="red") {% endmacro %} -{% macro config_dynamic_object(table, object) %} -{# Generate another nesting group in case table holds two types of objects #} +{% macro gen_cfg_dyn_obj(table, object) %} +{# Generate another nested group in case table holds two types of objects #} {% if table["dynamic-objects"]|length > 1 %} {% set group = table.name + "_" + object.name %} @{{ table.name }}.group(name="{{ cli_name(object.name) }}", @@ -406,10 +443,10 @@ def {{ group }}(): {% set group = table.name %} {% endif %} -{{ config_dynamic_object_add(group, table, object) }} -{{ config_dynamic_object_update(group, table, object) }} -{{ config_dynamic_object_delete(group, table, object) }} -{{ config_object_list_update_all(group, table, object) }} +{{ gen_cfg_dyn_obj_add(group, table, object) }} +{{ gen_cfg_dyn_obj_update(group, table, object) }} +{{ gen_cfg_dyn_obj_delete(group, table, object) }} +{{ gen_cfg_obj_list_update_all(group, table, object) }} {% endmacro %} @@ -423,13 +460,13 @@ def {{ table.name }}(): {% if "static-objects" in table %} {% for object in table["static-objects"] %} -{{ config_static_object(table, object) }} +{{ gen_cfg_static_obj(table, object) }} {% endfor %} {% endif %} {% if "dynamic-objects" in table %} {% for object in table["dynamic-objects"] %} -{{ config_dynamic_object(table, object) }} +{{ gen_cfg_dyn_obj(table, object) }} {% endfor %} {% endif %} diff --git a/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 b/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 index aad5fc539c..6ee27f2013 100644 --- a/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 +++ b/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 @@ -94,10 +94,19 @@ Result: ), {%- endfor %} {%- for group, attrs in attrs|selectattr("group", "defined")|groupby("group") %} +{%- if group == "" %} +{%- for attr in attrs %} + format_attr_value( + {{ entry }}, + {{ attr }} + ), +{%- endfor %} +{%- else %} format_group_value( {{ entry }}, {{ attrs }} ), +{%- endif %} {%- endfor %} ] {% endmacro %} @@ -128,7 +137,13 @@ Result: "{{ column_name(attr.name) }}", {% endfor -%} {% for group, attrs in attrs|selectattr("group", "defined")|groupby("group") -%} +{%- if group == "" %} +{% for attr in attrs -%} + "{{ column_name(attr.name) }}", +{% endfor -%} +{%- else %} "{{ column_name(group) }}", +{%- endif %} {% endfor -%} ] {% endmacro %} @@ -148,7 +163,7 @@ def TABLE_NAME(): @click.group(name="{{ cli_name(table.name) }}", cls=clicommon.AliasedGroup) def {{ table.name }}(): - """ {{ table.description }}""" + """ {{ table.description }} """ pass @@ -196,12 +211,13 @@ def {{ table.name }}(): {% set name = table.name %} {% endif %} +{# Generate an implementation to display table. #} @{{ group }}.group(name="{{ cli_name(name) }}", cls=clicommon.AliasedGroup, invoke_without_command=True) @clicommon.pass_db def {{ name }}(db): - """ {{ object.description }} """ + """ {{ object.description }} [Callable command group] """ header = {{ gen_header(object["keys"] + object.attrs) }} body = [] From 5c4e219107eb2ac13553fb825e1da02631418754 Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Mon, 24 May 2021 16:14:19 +0000 Subject: [PATCH 140/173] Fixed comments, added list_handler() where needed Signed-off-by: Vadym Hlushko --- sonic_cli_gen/yang_parser.py | 134 +++++++++++++++++------------------ 1 file changed, 66 insertions(+), 68 deletions(-) diff --git a/sonic_cli_gen/yang_parser.py b/sonic_cli_gen/yang_parser.py index 177fa5478e..0fdbb7fceb 100644 --- a/sonic_cli_gen/yang_parser.py +++ b/sonic_cli_gen/yang_parser.py @@ -94,21 +94,19 @@ def _init_yang_module_and_containers(self): self.y_table_containers = self.y_top_level_container.get('container') def _find_yang_model_in_yjson_obj(self) -> OrderedDict: - """ Find provided YANG model inside yJson object, - yJson object contain all yang-models parsed from directory - /usr/local/yang-models + """ Find provided YANG model inside the yJson object, + the yJson object contain all yang-models parsed from directory - /usr/local/yang-models Returns: reference to yang_model_name """ - # TODO: consider to check yJson type for yang_model in self.conf_mgmt.sy.yJson: if yang_model.get('module').get('@name') == self.yang_model_name: return yang_model.get('module') def parse_yang_model(self) -> dict: - """ Parse provided YANG model - and save output to self.yang_2_dict object + """ Parse provided YANG model and save the output to self.yang_2_dict object Returns: parsed YANG model in dictionary format @@ -117,9 +115,10 @@ def parse_yang_model(self) -> dict: self._init_yang_module_and_containers() self.yang_2_dict['tables'] = list() - # determine how many (1 or more) containers a YANG model have after 'top level' container - # 'table' container it is a container that goes after 'top level' container - self.yang_2_dict['tables'] += list_handler(self.y_table_containers, lambda e: on_table_container(self.y_module, e, self.conf_mgmt)) + # determine how many (1 or more) containers a YANG model have after the 'top level' container + # 'table' container goes after the 'top level' container + self.yang_2_dict['tables'] += list_handler(self.y_table_containers, + lambda e: on_table_container(self.y_module, e, self.conf_mgmt)) return self.yang_2_dict @@ -128,6 +127,7 @@ def parse_yang_model(self) -> dict: def list_handler(y_entity, callback) -> List[Dict]: """ Determine if the type of entity is a list, if so - call the callback for every list element """ + if isinstance(y_entity, list): return [callback(e) for e in y_entity] else: @@ -151,20 +151,20 @@ def on_table_container(y_module: OrderedDict, tbl_container: OrderedDict, conf_m 'description': get_description(tbl_container) } - # determine if 'table container' have a 'list' entity + # determine if 'table container' has a 'list' entity if tbl_container.get('list') is None: y2d_elem['static-objects'] = list() - # 'object' container goes after 'table' container - obj_container = tbl_container.get('container') + # 'object' container goes after the 'table' container # 'object' container have 2 types - list (like sonic-flex_counter.yang) and NOT list (like sonic-device_metadata.yang) - y2d_elem['static-objects'] += list_handler(obj_container, lambda e: on_object_container(y_module, e, conf_mgmt, is_list = False)) + y2d_elem['static-objects'] += list_handler(tbl_container.get('container'), + lambda e: on_object_entity(y_module, e, conf_mgmt, is_list = False)) else: y2d_elem['dynamic-objects'] = list() - tbl_container_lists = tbl_container.get('list') # 'container' can have more than 1 'list' entity - y2d_elem['dynamic-objects'] += list_handler(tbl_container_lists, lambda e: on_object_container(y_module, e, conf_mgmt, is_list = True)) + y2d_elem['dynamic-objects'] += list_handler(tbl_container.get('list'), + lambda e: on_object_entity(y_module, e, conf_mgmt, is_list = True)) # move 'keys' elements from 'attrs' to 'keys' change_dyn_obj_struct(y2d_elem['dynamic-objects']) @@ -172,9 +172,9 @@ def on_table_container(y_module: OrderedDict, tbl_container: OrderedDict, conf_m return y2d_elem -def on_object_container(y_module: OrderedDict, y_container: OrderedDict, conf_mgmt, is_list: bool) -> dict: - """ Parse a 'object container'. - 'Object container' represent OBJECT inside Config DB schema: +def on_object_entity(y_module: OrderedDict, y_entity: OrderedDict, conf_mgmt: ConfigMgmt, is_list: bool) -> dict: + """ Parse a 'object' entity, it could be a 'container' or a 'list' + 'Object' entity represent OBJECT inside Config DB schema: { "TABLE": { "OBJECT": { @@ -185,32 +185,32 @@ def on_object_container(y_module: OrderedDict, y_container: OrderedDict, conf_mg Args: y_module: reference to 'module' - y_container: reference to 'object container' + y_entity: reference to 'object' entity conf_mgmt: reference to ConfigMgmt class instance, it have yJson object which contain all parsed YANG models - is_list: boolean flag to determine if container has 'list' + is_list: boolean flag to determine if a 'list' was passed Returns: element for y2d_elem['static-objects'] OR y2d_elem['dynamic-objects'] """ - if y_container is None: + if y_entity is None: return {} obj_elem = { - 'name': y_container.get('@name'), - 'description': get_description(y_container), + 'name': y_entity.get('@name'), + 'description': get_description(y_entity), 'attrs': list() } if is_list: - obj_elem['keys'] = get_list_keys(y_container) + obj_elem['keys'] = get_list_keys(y_entity) attrs_list = list() # grouping_name is empty because 'grouping' is not used so far - attrs_list.extend(get_leafs(y_container, grouping_name = '')) - attrs_list.extend(get_leaf_lists(y_container, grouping_name = '')) - attrs_list.extend(get_choices(y_module, y_container, conf_mgmt, grouping_name = '')) - attrs_list.extend(get_uses(y_module, y_container, conf_mgmt)) + attrs_list.extend(get_leafs(y_entity, grouping_name = '')) + attrs_list.extend(get_leaf_lists(y_entity, grouping_name = '')) + attrs_list.extend(get_choices(y_module, y_entity, conf_mgmt, grouping_name = '')) + attrs_list.extend(get_uses(y_module, y_entity, conf_mgmt)) obj_elem['attrs'] = attrs_list @@ -219,7 +219,7 @@ def on_object_container(y_module: OrderedDict, y_container: OrderedDict, conf_mg def on_uses(y_module: OrderedDict, y_uses, conf_mgmt: ConfigMgmt) -> list: """ Parse a YANG 'uses' entities - 'uses' refearing to 'grouping' YANG entity + 'uses' referring to 'grouping' YANG entity Args: y_module: reference to 'module' @@ -282,18 +282,18 @@ def on_choices(y_module: OrderedDict, y_choices, conf_mgmt: ConfigMgmt, grouping return ret_attrs -def on_choice_cases(y_module: OrderedDict, y_cases: list, conf_mgmt: ConfigMgmt, grouping_name: str) -> list: - """ Parse a single YANG 'case' entity from 'choice' entity - 'case' element can have inside - 'leaf', 'leaf-list', 'uses' +def on_choice_cases(y_module: OrderedDict, y_cases, conf_mgmt: ConfigMgmt, grouping_name: str) -> list: + """ Parse a single YANG 'case' entity from the 'choice' entity. + The 'case' element can has inside - 'leaf', 'leaf-list', 'uses' Args: y_module: reference to 'module' y_cases: reference to 'case' conf_mgmt: reference to ConfigMgmt class instance, it have yJson object which contain all parsed YANG model - grouping_name: if YANG entity contain 'uses', this arg represent 'grouping' name + grouping_name: if YANG entity contain 'uses', this argument represent 'grouping' name Returns: - element for obj_elem['attrs'], 'attrs' contain a parsed 'leafs' + element for the obj_elem['attrs'], the 'attrs' contain a parsed 'leafs' """ ret_attrs = list() @@ -314,32 +314,26 @@ def on_leafs(y_leafs, grouping_name, is_leaf_list: bool) -> list: Args: y_leafs: reference to all 'leaf' elements - grouping_name: if YANG entity contain 'uses', this arg represent 'grouping' name - is_leaf_list: boolean to determine if 'leaf-list' was passed in 'y_leafs' arg + grouping_name: if YANG entity contain 'uses', this argument represent the 'grouping' name + is_leaf_list: boolean to determine if a 'leaf-list' was passed as 'y_leafs' argument Returns: list of parsed 'leaf' elements """ ret_attrs = list() # The YANG 'container' entity may have only 1 'leaf' element OR a list of 'leaf' elements - if isinstance(y_leafs, list): - for leaf in y_leafs: - attr = on_leaf(leaf, is_leaf_list, grouping_name) - ret_attrs.append(attr) - else: - attr = on_leaf(y_leafs, is_leaf_list, grouping_name) - ret_attrs.append(attr) + ret_attrs += list_handler(y_leafs, lambda e: on_leaf(e, grouping_name, is_leaf_list)) return ret_attrs -def on_leaf(leaf: OrderedDict, is_leaf_list: bool, grouping_name: str) -> dict: +def on_leaf(leaf: OrderedDict, grouping_name: str, is_leaf_list: bool) -> dict: """ Parse a single 'leaf' element Args: leaf: reference to a 'leaf' entity - grouping_name: if YANG entity contain 'uses', this arg represent 'grouping' name - is_leaf_list: boolean to determine if 'leaf-list' was passed in 'y_leafs' arg + grouping_name: if YANG entity contain 'uses', this argument represent 'grouping' name + is_leaf_list: boolean to determine if 'leaf-list' was passed in 'y_leafs' argument Returns: parsed 'leaf' element """ @@ -356,7 +350,7 @@ def on_leaf(leaf: OrderedDict, is_leaf_list: bool, grouping_name: str) -> dict: #----------------------GETERS-------------------------# def get_mandatory(y_leaf: OrderedDict) -> bool: - """ Parse 'mandatory' statement for 'leaf' + """ Parse the 'mandatory' statement for a 'leaf' Args: y_leaf: reference to a 'leaf' entity @@ -371,7 +365,7 @@ def get_mandatory(y_leaf: OrderedDict) -> bool: def get_description(y_entity: OrderedDict) -> str: - """ Parse 'description' entity from any YANG element + """ Parse the 'description' entity from any YANG element Args: y_entity: reference to YANG 'container' OR 'list' OR 'leaf' ... @@ -386,11 +380,11 @@ def get_description(y_entity: OrderedDict) -> str: def get_leafs(y_entity: OrderedDict, grouping_name: str) -> list: - """ Check if YANG entity have 'leafs', if so call handler + """ Check if the YANG entity have 'leafs', if so call handler Args: y_entity: reference YANG 'container' or 'list' or 'choice' or 'uses' - grouping_name: if YANG entity contain 'uses', this arg represent 'grouping' name + grouping_name: if YANG entity contain 'uses', this argument represent 'grouping' name Returns: list of parsed 'leaf' elements """ @@ -402,11 +396,11 @@ def get_leafs(y_entity: OrderedDict, grouping_name: str) -> list: def get_leaf_lists(y_entity: OrderedDict, grouping_name: str) -> list: - """ Check if YANG entity have 'leaf-list', if so call handler + """ Check if the YANG entity have 'leaf-list', if so call handler Args: y_entity: reference YANG 'container' or 'list' or 'choice' or 'uses' - grouping_name: if YANG entity contain 'uses', this arg represent 'grouping' name + grouping_name: if YANG entity contain 'uses', this argument represent 'grouping' name Returns: list of parsed 'leaf-list' elements """ @@ -418,14 +412,14 @@ def get_leaf_lists(y_entity: OrderedDict, grouping_name: str) -> list: def get_choices(y_module: OrderedDict, y_entity: OrderedDict, conf_mgmt: ConfigMgmt, grouping_name: str) -> list: - """ Check if YANG entity have 'choice', if so call handler + """ Check if the YANG entity have 'choice', if so call handler Args: y_module: reference to 'module' y_entity: reference YANG 'container' or 'list' or 'choice' or 'uses' conf_mgmt: reference to ConfigMgmt class instance, it have yJson object which contain all parsed YANG model - grouping_name: if YANG entity contain 'uses', this arg represent 'grouping' name + grouping_name: if YANG entity contain 'uses', this argument represent 'grouping' name Returns: list of parsed elements inside 'choice' """ @@ -437,7 +431,7 @@ def get_choices(y_module: OrderedDict, y_entity: OrderedDict, conf_mgmt: ConfigM def get_uses(y_module: OrderedDict, y_entity: OrderedDict, conf_mgmt: ConfigMgmt) -> list: - """ Check if YANG entity have 'uses', if so call handler + """ Check if the YANG entity have 'uses', if so call handler Args: y_module: reference to 'module' @@ -455,7 +449,7 @@ def get_uses(y_module: OrderedDict, y_entity: OrderedDict, conf_mgmt: ConfigMgmt def get_all_grouping(y_module: OrderedDict, y_uses: OrderedDict, conf_mgmt: ConfigMgmt) -> list: - """ Get all 'grouping' entities that is referenced by 'uses' in current YANG model + """ Get all the 'grouping' entities that was referenced by 'uses' in current YANG model Args: y_module: reference to 'module' @@ -467,14 +461,14 @@ def get_all_grouping(y_module: OrderedDict, y_uses: OrderedDict, conf_mgmt: Conf """ ret_grouping = list() + # prefix_list needed to find what YANG model was imported prefix_list = get_import_prefixes(y_uses) # in case if 'grouping' located in the same YANG model local_grouping = y_module.get('grouping') if local_grouping is not None: if isinstance(local_grouping, list): - for group in local_grouping: - ret_grouping.append(group) + ret_grouping.extend(local_grouping) else: ret_grouping.append(local_grouping) @@ -502,17 +496,16 @@ def get_grouping_from_another_yang_model(yang_model_name: str, conf_mgmt) -> lis it have yJson object which contain all parsed YANG models Returns: - list 'grouping' entities + list of 'grouping' entities """ ret_grouping = list() - for i in range(len(conf_mgmt.sy.yJson)): - if (conf_mgmt.sy.yJson[i].get('module').get('@name') == yang_model_name): - grouping = conf_mgmt.sy.yJson[i].get('module').get('grouping') + for yang_model in conf_mgmt.sy.yJson: + if (yang_model.get('module').get('@name') == yang_model_name): + grouping = yang_model.get('module').get('grouping') if isinstance(grouping, list): - for group in grouping: - ret_grouping.append(group) + ret_grouping.extend(grouping) else: ret_grouping.append(grouping) @@ -550,8 +543,8 @@ def get_import_prefixes(y_uses: OrderedDict) -> list: def trim_uses_prefixes(y_uses) -> list: - """ Trim prefixes from 'uses' YANG entities. - If YANG 'grouping' was imported from another YANG file, it use 'prefix' before 'grouping' name: + """ Trim prefixes from the 'uses' YANG entities. + If the YANG 'grouping' was imported from another YANG file, it use the 'prefix' before the 'grouping' name: { uses sgrop:endpoint; } @@ -559,6 +552,9 @@ def trim_uses_prefixes(y_uses) -> list: Args: y_uses: reference to 'uses' + + Returns: + list of 'uses' without 'prefixes' """ prefixes = get_import_prefixes(y_uses) @@ -574,16 +570,18 @@ def trim_uses_prefixes(y_uses) -> list: def get_list_keys(y_list: OrderedDict) -> list: - """ Parse YANG 'keys' - If YANG have 'list', inside the list exist 'keys' + """ Parse YANG the 'key' entity. + If YANG model has a 'list' entity, inside the 'list' there is 'key' entity. + 'key' - whitespace separeted list of 'leafs' Args: - y_list: reference to 'list' + y_list: reference to the 'list' Returns: list of parsed keys """ ret_list = list() + keys = y_list.get('key').get('@value').split() for k in keys: key = { 'name': k } From 6de672c5096fe0f2ed66d185f3a3dd80ca1ad61d Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Mon, 24 May 2021 16:35:21 +0000 Subject: [PATCH 141/173] Added logger to CliGenerator class constructor Signed-off-by: Vadym Hlushko --- sonic_cli_gen/generator.py | 20 +++++++++++++------- sonic_cli_gen/main.py | 7 ++++++- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/sonic_cli_gen/generator.py b/sonic_cli_gen/generator.py index 48142a9e7b..5a4f1be93d 100644 --- a/sonic_cli_gen/generator.py +++ b/sonic_cli_gen/generator.py @@ -9,19 +9,25 @@ class CliGenerator: """ SONiC CLI generator. This class provides public API for sonic-cli-gen python library. It can generate config, - show CLI plugins + show CLI plugins. + + Attributes: + loader: the loaded j2 templates + env: j2 central object + logger: logger """ - def __init__(self): + def __init__(self, logger): """ Initialize CliGenerator. """ self.loader = jinja2.FileSystemLoader(['/usr/share/sonic/templates/sonic-cli-gen/']) self.env = jinja2.Environment(loader=self.loader) + self.logger = logger def generate_cli_plugin(self, cli_group, plugin_name): """ Generate click CLI plugin and put it to: - /usr/local/lib/python3.7/dist-packages//plugins/auto/ + /usr/local/lib//dist-packages//plugins/auto/ """ parser = YangParser(yang_model_name=plugin_name, @@ -34,19 +40,19 @@ def generate_cli_plugin(self, cli_group, plugin_name): template = self.env.get_template(cli_group + '.py.j2') with open(plugin_path, 'w') as plugin_py: plugin_py.write(template.render(yang_dict)) - print('\nAuto-generation successful!\nLocation: {}'.format(plugin_path)) + self.logger.info(' Auto-generation successful! Location: {}'.format(plugin_path)) def remove_cli_plugin(self, cli_group, plugin_name): """ Remove CLI plugin from directory: - /usr/local/lib/python3.7/dist-packages//plugins/auto/ + /usr/local/lib//dist-packages//plugins/auto/ """ plugin_path = get_cli_plugin_path(cli_group, plugin_name + '_yang.py') if os.path.exists(plugin_path): os.remove(plugin_path) - print('{} was removed.'.format(plugin_path)) + self.logger.info(' {} was removed.'.format(plugin_path)) else: - print('Path {} doest NOT exist!'.format(plugin_path)) + self.logger.warning(' Path {} doest NOT exist!'.format(plugin_path)) def get_cli_plugin_path(command, plugin_name): diff --git a/sonic_cli_gen/main.py b/sonic_cli_gen/main.py index f23249d354..8f4de9a213 100644 --- a/sonic_cli_gen/main.py +++ b/sonic_cli_gen/main.py @@ -1,8 +1,13 @@ #!/usr/bin/env python +import sys import click +import logging from sonic_cli_gen.generator import CliGenerator +logger = logging.getLogger('sonic-cli-gen') +logging.basicConfig(stream=sys.stdout, level=logging.INFO) + @click.group() @click.pass_context def cli(ctx): @@ -15,7 +20,7 @@ def cli(ctx): """ context = { - 'gen': CliGenerator() + 'gen': CliGenerator(logger) } ctx.obj = context From 1584fe3d767d00d9158d735f97dd2e277c241d77 Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Tue, 25 May 2021 08:16:10 +0000 Subject: [PATCH 142/173] pep8 for generator.py Signed-off-by: Vadym Hlushko --- sonic_cli_gen/generator.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/sonic_cli_gen/generator.py b/sonic_cli_gen/generator.py index 5a4f1be93d..4f48b0201a 100644 --- a/sonic_cli_gen/generator.py +++ b/sonic_cli_gen/generator.py @@ -6,6 +6,9 @@ from sonic_cli_gen.yang_parser import YangParser +templates_path = '/usr/share/sonic/templates/sonic-cli-gen/' + + class CliGenerator: """ SONiC CLI generator. This class provides public API for sonic-cli-gen python library. It can generate config, @@ -20,11 +23,10 @@ class CliGenerator: def __init__(self, logger): """ Initialize CliGenerator. """ - self.loader = jinja2.FileSystemLoader(['/usr/share/sonic/templates/sonic-cli-gen/']) + self.loader = jinja2.FileSystemLoader(templates_path) self.env = jinja2.Environment(loader=self.loader) self.logger = logger - def generate_cli_plugin(self, cli_group, plugin_name): """ Generate click CLI plugin and put it to: /usr/local/lib//dist-packages//plugins/auto/ @@ -34,7 +36,8 @@ def generate_cli_plugin(self, cli_group, plugin_name): config_db_path='configDB', allow_tbl_without_yang=True, debug=False) - # yang_dict will be used as an input for templates located in - /usr/share/sonic/templates/sonic-cli-gen/ + # yang_dict will be used as an input for templates located in + # /usr/share/sonic/templates/sonic-cli-gen/ yang_dict = parser.parse_yang_model() plugin_path = get_cli_plugin_path(cli_group, plugin_name + '_yang.py') template = self.env.get_template(cli_group + '.py.j2') @@ -42,7 +45,6 @@ def generate_cli_plugin(self, cli_group, plugin_name): plugin_py.write(template.render(yang_dict)) self.logger.info(' Auto-generation successful! Location: {}'.format(plugin_path)) - def remove_cli_plugin(self, cli_group, plugin_name): """ Remove CLI plugin from directory: /usr/local/lib//dist-packages//plugins/auto/ @@ -52,7 +54,7 @@ def remove_cli_plugin(self, cli_group, plugin_name): os.remove(plugin_path) self.logger.info(' {} was removed.'.format(plugin_path)) else: - self.logger.warning(' Path {} doest NOT exist!'.format(plugin_path)) + self.logger.info(' Path {} doest NOT exist!'.format(plugin_path)) def get_cli_plugin_path(command, plugin_name): From 8f27d03c92bd44573c98313f0e99a9d6c7a38ed9 Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Tue, 25 May 2021 08:57:25 +0000 Subject: [PATCH 143/173] pep8 for the rest of files Signed-off-by: Vadym Hlushko --- sonic_cli_gen/main.py | 13 +- sonic_cli_gen/yang_parser.py | 244 ++++++++++++++++++++++------------- 2 files changed, 164 insertions(+), 93 deletions(-) diff --git a/sonic_cli_gen/main.py b/sonic_cli_gen/main.py index 8f4de9a213..bfcd301aed 100644 --- a/sonic_cli_gen/main.py +++ b/sonic_cli_gen/main.py @@ -8,6 +8,7 @@ logger = logging.getLogger('sonic-cli-gen') logging.basicConfig(stream=sys.stdout, level=logging.INFO) + @click.group() @click.pass_context def cli(ctx): @@ -26,23 +27,23 @@ def cli(ctx): @cli.command() -@click.argument('cli_group', type = click.Choice(['config', 'show'])) -@click.argument('yang_model_name', type = click.STRING) +@click.argument('cli_group', type=click.Choice(['config', 'show'])) +@click.argument('yang_model_name', type=click.STRING) @click.pass_context def generate(ctx, cli_group, yang_model_name): """ Generate click CLI plugin. """ - ctx.obj['gen'].generate_cli_plugin(cli_group = cli_group, plugin_name = yang_model_name) + ctx.obj['gen'].generate_cli_plugin(cli_group, yang_model_name) @cli.command() -@click.argument('cli_group', type = click.Choice(['config', 'show'])) -@click.argument('yang_model_name', type = click.STRING) +@click.argument('cli_group', type=click.Choice(['config', 'show'])) +@click.argument('yang_model_name', type=click.STRING) @click.pass_context def remove(ctx, cli_group, yang_model_name): """ Remove generated click CLI plugin from. """ - ctx.obj['gen'].remove_cli_plugin(cli_group = cli_group, plugin_name = yang_model_name) + ctx.obj['gen'].remove_cli_plugin(cli_group, yang_model_name) if __name__ == '__main__': diff --git a/sonic_cli_gen/yang_parser.py b/sonic_cli_gen/yang_parser.py index 0fdbb7fceb..23e420987c 100644 --- a/sonic_cli_gen/yang_parser.py +++ b/sonic_cli_gen/yang_parser.py @@ -2,23 +2,28 @@ from collections import OrderedDict from config.config_mgmt import ConfigMgmt - from typing import List, Dict yang_guidelines_link = 'https://github.com/Azure/SONiC/blob/master/doc/mgmt/SONiC_YANG_Model_Guidelines.md' + class YangParser: """ YANG model parser Attributes: yang_model_name: Name of the YANG model file - conf_mgmt: Instance of Config Mgmt class to help parse YANG models - y_module: Reference to 'module' entity from YANG model file - y_top_level_container: Reference to top level 'container' entity from YANG model file - y_table_containers: Reference to 'container' entities from YANG model file - that represent Config DB tables - yang_2_dict: dictionary created from YANG model file that represent Config DB schema. - In case if YANG model has a 'list' entity: + conf_mgmt: Instance of Config Mgmt class to + help parse YANG models + y_module: Reference to 'module' entity + from YANG model file + y_top_level_container: Reference to top level 'container' + entity from YANG model file + y_table_containers: Reference to 'container' entities + from YANG model file that represent Config DB tables + yang_2_dict: dictionary created from YANG model file that + represent Config DB schema. + + Below the 'yang_2_dict' obj in case if YANG model has a 'list' entity: { 'tables': [{ 'name': 'value', @@ -46,7 +51,9 @@ class YangParser: ], }] } - In case if YANG model does NOT have a 'list' entity, it has the same structure as above, but 'dynamic-objects' changed to 'static-objects' and have no 'keys' + In case if YANG model does NOT have a 'list' entity, + it has the same structure as above, but 'dynamic-objects' + changed to 'static-objects' and have no 'keys' """ def __init__(self, @@ -62,12 +69,12 @@ def __init__(self, self.yang_2_dict = dict() try: - self.conf_mgmt = ConfigMgmt(source=config_db_path, - debug=debug, - allowTablesWithoutYang=allow_tbl_without_yang) + self.conf_mgmt = ConfigMgmt(config_db_path, + debug, + allow_tbl_without_yang) except Exception as e: raise Exception("Failed to load the {} class".format(str(e))) - + def _init_yang_module_and_containers(self): """ Initialize inner class variables: self.y_module @@ -84,18 +91,23 @@ def _init_yang_module_and_containers(self): raise Exception('The YANG model {} is NOT exist'.format(self.yang_model_name)) if self.y_module.get('container') is None: - raise Exception('The YANG model {} does NOT have "top level container" element\ - Please follow the SONiC YANG model guidelines:\n{}'.format(self.yang_model_name, yang_guidelines_link)) + raise Exception('The YANG model {} does NOT have\ + "top level container" element\ + Please follow the SONiC YANG model guidelines:\ + \n{}'.format(self.yang_model_name, yang_guidelines_link)) self.y_top_level_container = self.y_module.get('container') if self.y_top_level_container.get('container') is None: - raise Exception('The YANG model {} does NOT have "container" element after "top level container"\ - Please follow the SONiC YANG model guidelines:\n{}'.format(self.yang_model_name, yang_guidelines_link)) + raise Exception('The YANG model {} does NOT have "container"\ + element after "top level container"\ + Please follow the SONiC YANG model guidelines:\ + \n{}'.format(self.yang_model_name, yang_guidelines_link)) self.y_table_containers = self.y_top_level_container.get('container') def _find_yang_model_in_yjson_obj(self) -> OrderedDict: """ Find provided YANG model inside the yJson object, - the yJson object contain all yang-models parsed from directory - /usr/local/yang-models + the yJson object contain all yang-models + parsed from directory - /usr/local/yang-models Returns: reference to yang_model_name @@ -106,7 +118,8 @@ def _find_yang_model_in_yjson_obj(self) -> OrderedDict: return yang_model.get('module') def parse_yang_model(self) -> dict: - """ Parse provided YANG model and save the output to self.yang_2_dict object + """ Parse provided YANG model and save + the output to self.yang_2_dict object Returns: parsed YANG model in dictionary format @@ -115,18 +128,21 @@ def parse_yang_model(self) -> dict: self._init_yang_module_and_containers() self.yang_2_dict['tables'] = list() - # determine how many (1 or more) containers a YANG model have after the 'top level' container + # determine how many (1 or more) containers a YANG model + # has after the 'top level' container # 'table' container goes after the 'top level' container self.yang_2_dict['tables'] += list_handler(self.y_table_containers, - lambda e: on_table_container(self.y_module, e, self.conf_mgmt)) + lambda e: on_table_container(self.y_module, e, self.conf_mgmt)) return self.yang_2_dict -#------------------------------HANDLERS--------------------------------# +# ------------------------------HANDLERS-------------------------------- # def list_handler(y_entity, callback) -> List[Dict]: - """ Determine if the type of entity is a list, if so - call the callback for every list element """ + """ Determine if the type of entity is a list, + if so - call the callback for every list element + """ if isinstance(y_entity, list): return [callback(e) for e in y_entity] @@ -134,7 +150,9 @@ def list_handler(y_entity, callback) -> List[Dict]: return [callback(y_entity)] -def on_table_container(y_module: OrderedDict, tbl_container: OrderedDict, conf_mgmt: ConfigMgmt) -> dict: +def on_table_container(y_module: OrderedDict, + tbl_container: OrderedDict, + conf_mgmt: ConfigMgmt) -> dict: """ Parse 'table' container, 'table' container goes after 'top level' container @@ -144,7 +162,7 @@ def on_table_container(y_module: OrderedDict, tbl_container: OrderedDict, conf_m conf_mgmt: reference to ConfigMgmt class instance, it have yJson object which contain all parsed YANG models Returns: - element for self.yang_2_dict['tables'] + element for self.yang_2_dict['tables'] """ y2d_elem = { 'name': tbl_container.get('@name'), @@ -156,15 +174,16 @@ def on_table_container(y_module: OrderedDict, tbl_container: OrderedDict, conf_m y2d_elem['static-objects'] = list() # 'object' container goes after the 'table' container - # 'object' container have 2 types - list (like sonic-flex_counter.yang) and NOT list (like sonic-device_metadata.yang) + # 'object' container have 2 types - list (like sonic-flex_counter.yang) + # and NOT list (like sonic-device_metadata.yang) y2d_elem['static-objects'] += list_handler(tbl_container.get('container'), - lambda e: on_object_entity(y_module, e, conf_mgmt, is_list = False)) + lambda e: on_object_entity(y_module, e, conf_mgmt, is_list=False)) else: y2d_elem['dynamic-objects'] = list() # 'container' can have more than 1 'list' entity y2d_elem['dynamic-objects'] += list_handler(tbl_container.get('list'), - lambda e: on_object_entity(y_module, e, conf_mgmt, is_list = True)) + lambda e: on_object_entity(y_module, e, conf_mgmt, is_list=True)) # move 'keys' elements from 'attrs' to 'keys' change_dyn_obj_struct(y2d_elem['dynamic-objects']) @@ -172,7 +191,10 @@ def on_table_container(y_module: OrderedDict, tbl_container: OrderedDict, conf_m return y2d_elem -def on_object_entity(y_module: OrderedDict, y_entity: OrderedDict, conf_mgmt: ConfigMgmt, is_list: bool) -> dict: +def on_object_entity(y_module: OrderedDict, + y_entity: OrderedDict, + conf_mgmt: ConfigMgmt, + is_list: bool) -> dict: """ Parse a 'object' entity, it could be a 'container' or a 'list' 'Object' entity represent OBJECT inside Config DB schema: { @@ -207,9 +229,9 @@ def on_object_entity(y_module: OrderedDict, y_entity: OrderedDict, conf_mgmt: Co attrs_list = list() # grouping_name is empty because 'grouping' is not used so far - attrs_list.extend(get_leafs(y_entity, grouping_name = '')) - attrs_list.extend(get_leaf_lists(y_entity, grouping_name = '')) - attrs_list.extend(get_choices(y_module, y_entity, conf_mgmt, grouping_name = '')) + attrs_list.extend(get_leafs(y_entity, grouping_name='')) + attrs_list.extend(get_leaf_lists(y_entity, grouping_name='')) + attrs_list.extend(get_choices(y_module, y_entity, conf_mgmt, grouping_name='')) attrs_list.extend(get_uses(y_module, y_entity, conf_mgmt)) obj_elem['attrs'] = attrs_list @@ -217,7 +239,9 @@ def on_object_entity(y_module: OrderedDict, y_entity: OrderedDict, conf_mgmt: Co return obj_elem -def on_uses(y_module: OrderedDict, y_uses, conf_mgmt: ConfigMgmt) -> list: +def on_uses(y_module: OrderedDict, + y_uses, + conf_mgmt: ConfigMgmt) -> list: """ Parse a YANG 'uses' entities 'uses' referring to 'grouping' YANG entity @@ -256,7 +280,10 @@ def on_uses(y_module: OrderedDict, y_uses, conf_mgmt: ConfigMgmt) -> list: return ret_attrs -def on_choices(y_module: OrderedDict, y_choices, conf_mgmt: ConfigMgmt, grouping_name: str) -> list: +def on_choices(y_module: OrderedDict, + y_choices, + conf_mgmt: ConfigMgmt, + grouping_name: str) -> list: """ Parse a YANG 'choice' entities Args: @@ -271,18 +298,24 @@ def on_choices(y_module: OrderedDict, y_choices, conf_mgmt: ConfigMgmt, grouping ret_attrs = list() - # the YANG model can have multiple 'choice' entities inside a 'container' or 'list' + # the YANG model can have multiple 'choice' entities + # inside a 'container' or 'list' if isinstance(y_choices, list): for choice in y_choices: - attrs = on_choice_cases(y_module, choice.get('case'), conf_mgmt, grouping_name) + attrs = on_choice_cases(y_module, choice.get('case'), + conf_mgmt, grouping_name) ret_attrs.extend(attrs) else: - ret_attrs = on_choice_cases(y_module, y_choices.get('case'), conf_mgmt, grouping_name) + ret_attrs = on_choice_cases(y_module, y_choices.get('case'), + conf_mgmt, grouping_name) return ret_attrs -def on_choice_cases(y_module: OrderedDict, y_cases, conf_mgmt: ConfigMgmt, grouping_name: str) -> list: +def on_choice_cases(y_module: OrderedDict, + y_cases, + conf_mgmt: ConfigMgmt, + grouping_name: str) -> list: """ Parse a single YANG 'case' entity from the 'choice' entity. The 'case' element can has inside - 'leaf', 'leaf-list', 'uses' @@ -290,10 +323,13 @@ def on_choice_cases(y_module: OrderedDict, y_cases, conf_mgmt: ConfigMgmt, group y_module: reference to 'module' y_cases: reference to 'case' conf_mgmt: reference to ConfigMgmt class instance, - it have yJson object which contain all parsed YANG model - grouping_name: if YANG entity contain 'uses', this argument represent 'grouping' name + it have yJson object which contain all + parsed YANG model + grouping_name: if YANG entity contain 'uses', + this argument represent 'grouping' name Returns: - element for the obj_elem['attrs'], the 'attrs' contain a parsed 'leafs' + element for the obj_elem['attrs'], the 'attrs' + contain a parsed 'leafs' """ ret_attrs = list() @@ -305,49 +341,58 @@ def on_choice_cases(y_module: OrderedDict, y_cases, conf_mgmt: ConfigMgmt, group ret_attrs.extend(get_uses(y_module, case, conf_mgmt)) else: raise Exception('It has no sense to using a single "case" element inside "choice" element') - + return ret_attrs -def on_leafs(y_leafs, grouping_name, is_leaf_list: bool) -> list: +def on_leafs(y_leafs, + grouping_name: str, + is_leaf_list: bool) -> list: """ Parse all the 'leaf' or 'leaf-list' elements Args: y_leafs: reference to all 'leaf' elements - grouping_name: if YANG entity contain 'uses', this argument represent the 'grouping' name - is_leaf_list: boolean to determine if a 'leaf-list' was passed as 'y_leafs' argument + grouping_name: if YANG entity contain 'uses', + this argument represent the 'grouping' name + is_leaf_list: boolean to determine if a 'leaf-list' + was passed as 'y_leafs' argument Returns: - list of parsed 'leaf' elements + list of parsed 'leaf' elements """ ret_attrs = list() - # The YANG 'container' entity may have only 1 'leaf' element OR a list of 'leaf' elements + # The YANG 'container' entity may have only 1 'leaf' + # element OR a list of 'leaf' elements ret_attrs += list_handler(y_leafs, lambda e: on_leaf(e, grouping_name, is_leaf_list)) return ret_attrs -def on_leaf(leaf: OrderedDict, grouping_name: str, is_leaf_list: bool) -> dict: +def on_leaf(leaf: OrderedDict, + grouping_name: str, + is_leaf_list: bool) -> dict: """ Parse a single 'leaf' element Args: leaf: reference to a 'leaf' entity - grouping_name: if YANG entity contain 'uses', this argument represent 'grouping' name - is_leaf_list: boolean to determine if 'leaf-list' was passed in 'y_leafs' argument + grouping_name: if YANG entity contain 'uses', + this argument represent 'grouping' name + is_leaf_list: boolean to determine if 'leaf-list' + was passed in 'y_leafs' argument Returns: parsed 'leaf' element """ - attr = { 'name': leaf.get('@name'), - 'description': get_description(leaf), - 'is-leaf-list': is_leaf_list, - 'is-mandatory': get_mandatory(leaf), - 'group': grouping_name} + attr = {'name': leaf.get('@name'), + 'description': get_description(leaf), + 'is-leaf-list': is_leaf_list, + 'is-mandatory': get_mandatory(leaf), + 'group': grouping_name} return attr -#----------------------GETERS-------------------------# +# ----------------------GETERS------------------------- # def get_mandatory(y_leaf: OrderedDict) -> bool: """ Parse the 'mandatory' statement for a 'leaf' @@ -379,14 +424,17 @@ def get_description(y_entity: OrderedDict) -> str: return '' -def get_leafs(y_entity: OrderedDict, grouping_name: str) -> list: +def get_leafs(y_entity: OrderedDict, + grouping_name: str) -> list: """ Check if the YANG entity have 'leafs', if so call handler Args: - y_entity: reference YANG 'container' or 'list' or 'choice' or 'uses' - grouping_name: if YANG entity contain 'uses', this argument represent 'grouping' name + y_entity: reference YANG 'container' or 'list' + or 'choice' or 'uses' + grouping_name: if YANG entity contain 'uses', + this argument represent 'grouping' name Returns: - list of parsed 'leaf' elements + list of parsed 'leaf' elements """ if y_entity.get('leaf') is not None: @@ -395,14 +443,17 @@ def get_leafs(y_entity: OrderedDict, grouping_name: str) -> list: return [] -def get_leaf_lists(y_entity: OrderedDict, grouping_name: str) -> list: +def get_leaf_lists(y_entity: OrderedDict, + grouping_name: str) -> list: """ Check if the YANG entity have 'leaf-list', if so call handler Args: - y_entity: reference YANG 'container' or 'list' or 'choice' or 'uses' - grouping_name: if YANG entity contain 'uses', this argument represent 'grouping' name + y_entity: reference YANG 'container' or 'list' + or 'choice' or 'uses' + grouping_name: if YANG entity contain 'uses', + this argument represent 'grouping' name Returns: - list of parsed 'leaf-list' elements + list of parsed 'leaf-list' elements """ if y_entity.get('leaf-list') is not None: @@ -411,15 +462,20 @@ def get_leaf_lists(y_entity: OrderedDict, grouping_name: str) -> list: return [] -def get_choices(y_module: OrderedDict, y_entity: OrderedDict, conf_mgmt: ConfigMgmt, grouping_name: str) -> list: +def get_choices(y_module: OrderedDict, + y_entity: OrderedDict, + conf_mgmt: ConfigMgmt, + grouping_name: str) -> list: """ Check if the YANG entity have 'choice', if so call handler Args: y_module: reference to 'module' - y_entity: reference YANG 'container' or 'list' or 'choice' or 'uses' + y_entity: reference YANG 'container' or 'list' + or 'choice' or 'uses' conf_mgmt: reference to ConfigMgmt class instance, - it have yJson object which contain all parsed YANG model - grouping_name: if YANG entity contain 'uses', this argument represent 'grouping' name + it have yJson object which contain all parsed YANG model + grouping_name: if YANG entity contain 'uses', + this argument represent 'grouping' name Returns: list of parsed elements inside 'choice' """ @@ -430,16 +486,20 @@ def get_choices(y_module: OrderedDict, y_entity: OrderedDict, conf_mgmt: ConfigM return [] -def get_uses(y_module: OrderedDict, y_entity: OrderedDict, conf_mgmt: ConfigMgmt) -> list: +def get_uses(y_module: OrderedDict, + y_entity: OrderedDict, + conf_mgmt: ConfigMgmt) -> list: """ Check if the YANG entity have 'uses', if so call handler Args: y_module: reference to 'module' - y_entity: reference YANG 'container' or 'list' or 'choice' or 'uses' + y_entity: reference YANG 'container' or 'list' + or 'choice' or 'uses' conf_mgmt: reference to ConfigMgmt class instance, - it have yJson object which contain all parsed YANG model + it have yJson object which contain all parsed YANG model Returns: - list of parsed elements inside 'grouping' that referenced by 'uses' + list of parsed elements inside 'grouping' + that referenced by 'uses' """ if y_entity.get('uses') is not None: @@ -448,14 +508,17 @@ def get_uses(y_module: OrderedDict, y_entity: OrderedDict, conf_mgmt: ConfigMgmt return [] -def get_all_grouping(y_module: OrderedDict, y_uses: OrderedDict, conf_mgmt: ConfigMgmt) -> list: - """ Get all the 'grouping' entities that was referenced by 'uses' in current YANG model +def get_all_grouping(y_module: OrderedDict, + y_uses: OrderedDict, + conf_mgmt: ConfigMgmt) -> list: + """ Get all the 'grouping' entities that was referenced + by 'uses' in current YANG model Args: y_module: reference to 'module' y_entity: reference to 'uses' conf_mgmt: reference to ConfigMgmt class instance, - it have yJson object which contain all parsed YANG model + it have yJson object which contain all parsed YANG model Returns: list of 'grouping' elements """ @@ -472,7 +535,8 @@ def get_all_grouping(y_module: OrderedDict, y_uses: OrderedDict, conf_mgmt: Conf else: ret_grouping.append(local_grouping) - # if prefix_list is NOT empty it means that 'grouping' was imported from another YANG model + # if prefix_list is NOT empty it means that 'grouping' + # was imported from another YANG model if prefix_list != []: for prefix in prefix_list: y_import = y_module.get('import') @@ -487,13 +551,14 @@ def get_all_grouping(y_module: OrderedDict, y_uses: OrderedDict, conf_mgmt: Conf return ret_grouping -def get_grouping_from_another_yang_model(yang_model_name: str, conf_mgmt) -> list: +def get_grouping_from_another_yang_model(yang_model_name: str, + conf_mgmt) -> list: """ Get the YANG 'grouping' entity Args: yang_model_name - YANG model to search conf_mgmt - reference to ConfigMgmt class instance, - it have yJson object which contain all parsed YANG models + it have yJson object which contain all parsed YANG models Returns: list of 'grouping' entities @@ -516,7 +581,7 @@ def get_import_prefixes(y_uses: OrderedDict) -> list: """ Parse 'import prefix' of YANG 'uses' entity Example: { - uses stypes:endpoint; + uses stypes:endpoint; } 'stypes' - prefix of imported YANG module. 'endpoint' - YANG 'grouping' entity name @@ -544,7 +609,8 @@ def get_import_prefixes(y_uses: OrderedDict) -> list: def trim_uses_prefixes(y_uses) -> list: """ Trim prefixes from the 'uses' YANG entities. - If the YANG 'grouping' was imported from another YANG file, it use the 'prefix' before the 'grouping' name: + If the YANG 'grouping' was imported from another + YANG file, it use the 'prefix' before the 'grouping' name: { uses sgrop:endpoint; } @@ -571,8 +637,9 @@ def trim_uses_prefixes(y_uses) -> list: def get_list_keys(y_list: OrderedDict) -> list: """ Parse YANG the 'key' entity. - If YANG model has a 'list' entity, inside the 'list' there is 'key' entity. - 'key' - whitespace separeted list of 'leafs' + If YANG model has a 'list' entity, inside the 'list' + there is 'key' entity. The 'key' - whitespace + separeted list of 'leafs' Args: y_list: reference to the 'list' @@ -584,7 +651,7 @@ def get_list_keys(y_list: OrderedDict) -> list: keys = y_list.get('key').get('@value').split() for k in keys: - key = { 'name': k } + key = {'name': k} ret_list.append(key) return ret_list @@ -592,10 +659,13 @@ def get_list_keys(y_list: OrderedDict) -> list: def change_dyn_obj_struct(dynamic_objects: list): """ Rearrange self.yang_2_dict['dynamic_objects'] structure. - If YANG model have a 'list' entity - inside the 'list' it has 'key' entity. - 'key' entity it is whitespace-separeted list of 'leafs', those 'leafs' was - parsed by 'on_leaf()' function and placed under 'attrs' in self.yang_2_dict['dynamic_objects'] - need to move 'leafs' from 'attrs' and put them to 'keys' section of elf.yang_2_dict['dynamic_objects'] + If YANG model have a 'list' entity - inside the 'list' + it has 'key' entity. The 'key' entity it is whitespace + separeted list of 'leafs', those 'leafs' was parsed by + 'on_leaf()' function and placed under 'attrs' in + self.yang_2_dict['dynamic_objects'] need to move 'leafs' + from 'attrs' and put them into 'keys' section of + self.yang_2_dict['dynamic_objects'] Args: dynamic_objects: reference to self.yang_2_dict['dynamic_objects'] From aa6f806f7294f7de46584a4a49a8466fbdda43df Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Tue, 25 May 2021 09:24:09 +0000 Subject: [PATCH 144/173] pep8 for tests files, added comments to YANG models Signed-off-by: Vadym Hlushko --- sonic_cli_gen/yang_parser.py | 2 +- tests/cli_autogen_input/sonic-1-list.yang | 3 + .../sonic-1-object-container.yang | 3 + .../sonic-1-table-container.yang | 2 + tests/cli_autogen_input/sonic-2-lists.yang | 5 + .../sonic-2-object-containers.yang | 4 + .../sonic-2-table-containers.yang | 3 + tests/cli_autogen_yang_parser_test.py | 114 ++++++++++++++---- 8 files changed, 111 insertions(+), 25 deletions(-) diff --git a/sonic_cli_gen/yang_parser.py b/sonic_cli_gen/yang_parser.py index 23e420987c..f0c737802b 100644 --- a/sonic_cli_gen/yang_parser.py +++ b/sonic_cli_gen/yang_parser.py @@ -196,7 +196,7 @@ def on_object_entity(y_module: OrderedDict, conf_mgmt: ConfigMgmt, is_list: bool) -> dict: """ Parse a 'object' entity, it could be a 'container' or a 'list' - 'Object' entity represent OBJECT inside Config DB schema: + 'Object' entity represent OBJECT in Config DB schema: { "TABLE": { "OBJECT": { diff --git a/tests/cli_autogen_input/sonic-1-list.yang b/tests/cli_autogen_input/sonic-1-list.yang index c7fc4ee824..79a6529b3d 100644 --- a/tests/cli_autogen_input/sonic-1-list.yang +++ b/tests/cli_autogen_input/sonic-1-list.yang @@ -6,12 +6,15 @@ module sonic-1-list { prefix s-1-list; container sonic-1-list { + /* sonic-1-list - top level container */ container TABLE_1 { + /* TABLE_1 - table container */ description "TABLE_1 description"; list TABLE_1_LIST { + /* TABLE_1 - object container */ description "TABLE_1_LIST description"; diff --git a/tests/cli_autogen_input/sonic-1-object-container.yang b/tests/cli_autogen_input/sonic-1-object-container.yang index d52b2a8caf..e28ef7f90a 100644 --- a/tests/cli_autogen_input/sonic-1-object-container.yang +++ b/tests/cli_autogen_input/sonic-1-object-container.yang @@ -6,12 +6,15 @@ module sonic-1-object-container { prefix s-1-object; container sonic-1-object-container { + /* sonic-1-object-container - top level container */ container TABLE_1 { + /* TABLE_1 - table container */ description "TABLE_1 description"; container OBJECT_1 { + /* OBJECT_1 - object container */ description "OBJECT_1 description"; } diff --git a/tests/cli_autogen_input/sonic-1-table-container.yang b/tests/cli_autogen_input/sonic-1-table-container.yang index 8963148158..58e7293c0d 100644 --- a/tests/cli_autogen_input/sonic-1-table-container.yang +++ b/tests/cli_autogen_input/sonic-1-table-container.yang @@ -6,8 +6,10 @@ module sonic-1-table-container { prefix s-1-table; container sonic-1-table-container { + /* sonic-1-table-container - top level container */ container TABLE_1 { + /* TABLE_1 - table container */ description "TABLE_1 description"; } diff --git a/tests/cli_autogen_input/sonic-2-lists.yang b/tests/cli_autogen_input/sonic-2-lists.yang index 2a4cd42fd9..b20200415b 100644 --- a/tests/cli_autogen_input/sonic-2-lists.yang +++ b/tests/cli_autogen_input/sonic-2-lists.yang @@ -6,12 +6,16 @@ module sonic-2-lists { prefix s-2-lists; container sonic-2-lists { + /* sonic-2-lists - top level container */ container TABLE_1 { + /* TALBE_1 - table container */ + description "TABLE_1 description"; list TABLE_1_LIST_1 { + /* TALBE_1_LIST_1 - object container */ description "TABLE_1_LIST_1 description"; @@ -23,6 +27,7 @@ module sonic-2-lists { } list TABLE_1_LIST_2 { + /* TALBE_1_LIST_2 - object container */ description "TABLE_1_LIST_2 description"; diff --git a/tests/cli_autogen_input/sonic-2-object-containers.yang b/tests/cli_autogen_input/sonic-2-object-containers.yang index 1aaaeb1a19..249faf4c89 100644 --- a/tests/cli_autogen_input/sonic-2-object-containers.yang +++ b/tests/cli_autogen_input/sonic-2-object-containers.yang @@ -6,17 +6,21 @@ module sonic-2-object-containers { prefix s-2-object; container sonic-2-object-containers { + /* sonic-2-object-containers - top level container */ container TABLE_1 { + /* TABLE_1 - table container */ description "FIRST_TABLE description"; container OBJECT_1 { + /* OBJECT_1 - object container */ description "OBJECT_1 description"; } container OBJECT_2 { + /* OBJECT_2 - object container */ description "OBJECT_2 description"; } diff --git a/tests/cli_autogen_input/sonic-2-table-containers.yang b/tests/cli_autogen_input/sonic-2-table-containers.yang index a3f13474b5..393512a313 100644 --- a/tests/cli_autogen_input/sonic-2-table-containers.yang +++ b/tests/cli_autogen_input/sonic-2-table-containers.yang @@ -6,13 +6,16 @@ module sonic-2-table-containers { prefix s-2-table; container sonic-2-table-containers { + /* sonic-2-table-containers - top level container */ container TABLE_1 { + /* TABLE_1 - table container */ description "TABLE_1 description"; } container TABLE_2 { + /* TABLE_2 - table container */ description "TABLE_2 description"; } diff --git a/tests/cli_autogen_yang_parser_test.py b/tests/cli_autogen_yang_parser_test.py index 67898503af..023b9fa6e1 100644 --- a/tests/cli_autogen_yang_parser_test.py +++ b/tests/cli_autogen_yang_parser_test.py @@ -26,6 +26,7 @@ 'sonic-grouping-2', ] + class TestYangParser: @classmethod def setup_class(cls): @@ -37,89 +38,154 @@ def setup_class(cls): def teardown_class(cls): logger.info("TEARDOWN") os.environ['UTILITIES_UNIT_TESTING'] = "0" - remove_yang_models_to_well_know_location() + remove_yang_models_from_well_know_location() def test_1_table_container(self): - template('sonic-1-table-container', assert_dictionaries.one_table_container) - + """ Test for 1 'table' container + 'table' container represent TABLE in Config DB schema: + { + "TABLE": { + "OBJECT": { + "attr": "value" + ... + } + } + } + """ + + template('sonic-1-table-container', + assert_dictionaries.one_table_container) + def test_2_table_containers(self): - template('sonic-2-table-containers', assert_dictionaries.two_table_containers) + """ Test for 2 'table' containers """ + + template('sonic-2-table-containers', + assert_dictionaries.two_table_containers) def test_1_object_container(self): - template('sonic-1-object-container', assert_dictionaries.one_object_container) + """ Test for 1 'object' container + 'object' container represent OBJECT in Config DB schema: + { + "TABLE": { + "OBJECT": { + "attr": "value" + ... + } + } + } + """ + + template('sonic-1-object-container', + assert_dictionaries.one_object_container) def test_2_object_containers(self): - template('sonic-2-object-containers', assert_dictionaries.two_object_containers) + """ Test for 2 'object' containers """ + + template('sonic-2-object-containers', + assert_dictionaries.two_object_containers) def test_1_list(self): + """ Test for 1 container that has inside + the YANG 'list' entity + """ + template('sonic-1-list', assert_dictionaries.one_list) def test_2_lists(self): + """ Test for 2 containers that have inside + the YANG 'list' entity + """ + template('sonic-2-lists', assert_dictionaries.two_lists) def test_static_object_complex_1(self): - """ Test object container with: 1 leaf, 1 leaf-list, 1 choice. + """ Test for the object container with: + 1 leaf, 1 leaf-list, 1 choice. """ - template('sonic-static-object-complex-1', assert_dictionaries.static_object_complex_1) + + template('sonic-static-object-complex-1', + assert_dictionaries.static_object_complex_1) def test_static_object_complex_2(self): - """ Test object container with: 2 leafs, 2 leaf-lists, 2 choices. + """ Test for object container with: + 2 leafs, 2 leaf-lists, 2 choices. """ - template('sonic-static-object-complex-2', assert_dictionaries.static_object_complex_2) + + template('sonic-static-object-complex-2', + assert_dictionaries.static_object_complex_2) def test_dynamic_object_complex_1(self): - """ Test object container with: 1 key, 1 leaf, 1 leaf-list, 1 choice. + """ Test object container with: + 1 key, 1 leaf, 1 leaf-list, 1 choice. """ - template('sonic-dynamic-object-complex-1', assert_dictionaries.dynamic_object_complex_1) + + template('sonic-dynamic-object-complex-1', + assert_dictionaries.dynamic_object_complex_1) def test_dynamic_object_complex_2(self): - """ Test object container with: 2 keys, 2 leafs, 2 leaf-list, 2 choice. + """ Test object container with: + 2 keys, 2 leafs, 2 leaf-list, 2 choice. """ - template('sonic-dynamic-object-complex-2', assert_dictionaries.dynamic_object_complex_2) + + template('sonic-dynamic-object-complex-2', + assert_dictionaries.dynamic_object_complex_2) def test_choice_complex(self): """ Test object container with choice that have complex strucutre: leafs, leaf-lists, multiple 'uses' from different files """ - template('sonic-choice-complex', assert_dictionaries.choice_complex) + + template('sonic-choice-complex', + assert_dictionaries.choice_complex) def test_grouping_complex(self): """ Test object container with muplitple 'uses' that using 'grouping' from different files. The used 'grouping' have a complex strucutre: leafs, leaf-lists, choices """ - template('sonic-grouping-complex', assert_dictionaries.grouping_complex) + + template('sonic-grouping-complex', + assert_dictionaries.grouping_complex) + def template(yang_model_name, correct_dict): - config_db_path = os.path.join(test_path, 'cli_autogen_input/config_db.json') - parser = YangParser(yang_model_name = yang_model_name, - config_db_path = config_db_path, - allow_tbl_without_yang = True, - debug = False) + config_db_path = os.path.join(test_path, + 'cli_autogen_input/config_db.json') + parser = YangParser(yang_model_name=yang_model_name, + config_db_path=config_db_path, + allow_tbl_without_yang=True, + debug=False) yang_dict = parser.parse_yang_model() pretty_log_debug(yang_dict) assert yang_dict == correct_dict + def move_yang_models_to_well_know_location(): """ Move a test YANG models to known location in order to be parsed by YangParser class """ for yang_model in test_yang_models: - src_path = os.path.join(test_path, 'cli_autogen_input', yang_model + '.yang') + src_path = os.path.join(test_path, + 'cli_autogen_input', + yang_model + '.yang') cmd = 'sudo cp {} {}'.format(src_path, yang_models_path) os.system(cmd) -def remove_yang_models_to_well_know_location(): + +def remove_yang_models_from_well_know_location(): """ Remove a test YANG models to known location in order to be parsed by YangParser class """ for yang_model in test_yang_models: - yang_model_path = os.path.join(yang_models_path, yang_model + '.yang') + yang_model_path = os.path.join(yang_models_path, + yang_model + '.yang') cmd = 'sudo rm {}'.format(yang_model_path) os.system(cmd) + def pretty_log_debug(dictionary): """ Pretty print of parsed dictionary """ for line in pprint.pformat(dictionary).split('\n'): logging.debug(line) + From 83ce9ee534a92713ee11a93e7ecf6f08d90f850d Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Tue, 25 May 2021 11:41:48 +0000 Subject: [PATCH 145/173] Added handler for 1 'choice' 'case' Signed-off-by: Vadym Hlushko --- sonic_cli_gen/yang_parser.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/sonic_cli_gen/yang_parser.py b/sonic_cli_gen/yang_parser.py index f0c737802b..df0382536f 100644 --- a/sonic_cli_gen/yang_parser.py +++ b/sonic_cli_gen/yang_parser.py @@ -259,10 +259,6 @@ def on_uses(y_module: OrderedDict, # trim prefixes in order to the next checks trim_uses_prefixes(y_uses) - # not sure if it can happend - if y_grouping == []: - raise Exception('Grouping NOT found') - # TODO: 'refine' support for group in y_grouping: if isinstance(y_uses, list): @@ -340,7 +336,9 @@ def on_choice_cases(y_module: OrderedDict, ret_attrs.extend(get_leaf_lists(case, grouping_name)) ret_attrs.extend(get_uses(y_module, case, conf_mgmt)) else: - raise Exception('It has no sense to using a single "case" element inside "choice" element') + ret_attrs.extend(get_leafs(y_cases, grouping_name)) + ret_attrs.extend(get_leaf_lists(y_cases, grouping_name)) + ret_attrs.extend(get_uses(y_module, y_cases, conf_mgmt)) return ret_attrs From 702be7427a732a9deaa8a7e6e0eb3748737c3730 Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Tue, 25 May 2021 11:47:20 +0000 Subject: [PATCH 146/173] Code style Signed-off-by: Vadym Hlushko --- tests/cli_autogen_yang_parser_test.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/tests/cli_autogen_yang_parser_test.py b/tests/cli_autogen_yang_parser_test.py index 023b9fa6e1..61c2dc98d9 100644 --- a/tests/cli_autogen_yang_parser_test.py +++ b/tests/cli_autogen_yang_parser_test.py @@ -115,7 +115,7 @@ def test_static_object_complex_2(self): assert_dictionaries.static_object_complex_2) def test_dynamic_object_complex_1(self): - """ Test object container with: + """ Test for object container with: 1 key, 1 leaf, 1 leaf-list, 1 choice. """ @@ -123,7 +123,7 @@ def test_dynamic_object_complex_1(self): assert_dictionaries.dynamic_object_complex_1) def test_dynamic_object_complex_2(self): - """ Test object container with: + """ Test for object container with: 2 keys, 2 leafs, 2 leaf-list, 2 choice. """ @@ -131,7 +131,8 @@ def test_dynamic_object_complex_2(self): assert_dictionaries.dynamic_object_complex_2) def test_choice_complex(self): - """ Test object container with choice that have complex strucutre: + """ Test for object container with the 'choice' + that have complex strucutre: leafs, leaf-lists, multiple 'uses' from different files """ @@ -139,8 +140,8 @@ def test_choice_complex(self): assert_dictionaries.choice_complex) def test_grouping_complex(self): - """ Test object container with muplitple 'uses' that using 'grouping' - from different files. The used 'grouping' have a complex strucutre: + """ Test for object container with multitple 'uses' that using 'grouping' + from different files. The used 'grouping' have a complex structure: leafs, leaf-lists, choices """ @@ -149,6 +150,8 @@ def test_grouping_complex(self): def template(yang_model_name, correct_dict): + """ General template for every test case """ + config_db_path = os.path.join(test_path, 'cli_autogen_input/config_db.json') parser = YangParser(yang_model_name=yang_model_name, @@ -164,6 +167,7 @@ def move_yang_models_to_well_know_location(): """ Move a test YANG models to known location in order to be parsed by YangParser class """ + for yang_model in test_yang_models: src_path = os.path.join(test_path, 'cli_autogen_input', @@ -176,6 +180,7 @@ def remove_yang_models_from_well_know_location(): """ Remove a test YANG models to known location in order to be parsed by YangParser class """ + for yang_model in test_yang_models: yang_model_path = os.path.join(yang_models_path, yang_model + '.yang') @@ -184,8 +189,8 @@ def remove_yang_models_from_well_know_location(): def pretty_log_debug(dictionary): - """ Pretty print of parsed dictionary - """ + """ Pretty print of parsed dictionary """ + for line in pprint.pformat(dictionary).split('\n'): logging.debug(line) From c433a4c7a231ad0486407daf5aad56be16c091df Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Wed, 26 May 2021 15:36:58 +0000 Subject: [PATCH 147/173] Fixed review comments Signed-off-by: Vadym Hlushko --- tests/cli_autogen_yang_parser_test.py | 36 +++++++++++++-------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/tests/cli_autogen_yang_parser_test.py b/tests/cli_autogen_yang_parser_test.py index 61c2dc98d9..9ed915c69b 100644 --- a/tests/cli_autogen_yang_parser_test.py +++ b/tests/cli_autogen_yang_parser_test.py @@ -32,13 +32,13 @@ class TestYangParser: def setup_class(cls): logger.info("SETUP") os.environ['UTILITIES_UNIT_TESTING'] = "1" - move_yang_models_to_well_know_location() + move_yang_models() @classmethod def teardown_class(cls): logger.info("TEARDOWN") os.environ['UTILITIES_UNIT_TESTING'] = "0" - remove_yang_models_from_well_know_location() + remove_yang_models() def test_1_table_container(self): """ Test for 1 'table' container @@ -53,13 +53,13 @@ def test_1_table_container(self): } """ - template('sonic-1-table-container', + base_test('sonic-1-table-container', assert_dictionaries.one_table_container) def test_2_table_containers(self): """ Test for 2 'table' containers """ - template('sonic-2-table-containers', + base_test('sonic-2-table-containers', assert_dictionaries.two_table_containers) def test_1_object_container(self): @@ -75,13 +75,13 @@ def test_1_object_container(self): } """ - template('sonic-1-object-container', + base_test('sonic-1-object-container', assert_dictionaries.one_object_container) def test_2_object_containers(self): """ Test for 2 'object' containers """ - template('sonic-2-object-containers', + base_test('sonic-2-object-containers', assert_dictionaries.two_object_containers) def test_1_list(self): @@ -89,21 +89,21 @@ def test_1_list(self): the YANG 'list' entity """ - template('sonic-1-list', assert_dictionaries.one_list) + base_test('sonic-1-list', assert_dictionaries.one_list) def test_2_lists(self): """ Test for 2 containers that have inside the YANG 'list' entity """ - template('sonic-2-lists', assert_dictionaries.two_lists) + base_test('sonic-2-lists', assert_dictionaries.two_lists) def test_static_object_complex_1(self): """ Test for the object container with: 1 leaf, 1 leaf-list, 1 choice. """ - template('sonic-static-object-complex-1', + base_test('sonic-static-object-complex-1', assert_dictionaries.static_object_complex_1) def test_static_object_complex_2(self): @@ -111,7 +111,7 @@ def test_static_object_complex_2(self): 2 leafs, 2 leaf-lists, 2 choices. """ - template('sonic-static-object-complex-2', + base_test('sonic-static-object-complex-2', assert_dictionaries.static_object_complex_2) def test_dynamic_object_complex_1(self): @@ -119,7 +119,7 @@ def test_dynamic_object_complex_1(self): 1 key, 1 leaf, 1 leaf-list, 1 choice. """ - template('sonic-dynamic-object-complex-1', + base_test('sonic-dynamic-object-complex-1', assert_dictionaries.dynamic_object_complex_1) def test_dynamic_object_complex_2(self): @@ -127,7 +127,7 @@ def test_dynamic_object_complex_2(self): 2 keys, 2 leafs, 2 leaf-list, 2 choice. """ - template('sonic-dynamic-object-complex-2', + base_test('sonic-dynamic-object-complex-2', assert_dictionaries.dynamic_object_complex_2) def test_choice_complex(self): @@ -136,7 +136,7 @@ def test_choice_complex(self): leafs, leaf-lists, multiple 'uses' from different files """ - template('sonic-choice-complex', + base_test('sonic-choice-complex', assert_dictionaries.choice_complex) def test_grouping_complex(self): @@ -145,12 +145,12 @@ def test_grouping_complex(self): leafs, leaf-lists, choices """ - template('sonic-grouping-complex', + base_test('sonic-grouping-complex', assert_dictionaries.grouping_complex) -def template(yang_model_name, correct_dict): - """ General template for every test case """ +def base_test(yang_model_name, correct_dict): + """ General logic for each test case """ config_db_path = os.path.join(test_path, 'cli_autogen_input/config_db.json') @@ -163,7 +163,7 @@ def template(yang_model_name, correct_dict): assert yang_dict == correct_dict -def move_yang_models_to_well_know_location(): +def move_yang_models(): """ Move a test YANG models to known location in order to be parsed by YangParser class """ @@ -176,7 +176,7 @@ def move_yang_models_to_well_know_location(): os.system(cmd) -def remove_yang_models_from_well_know_location(): +def remove_yang_models(): """ Remove a test YANG models to known location in order to be parsed by YangParser class """ From 6244efefb788da29ee05c599ef7bbd1e2ed1d5b7 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Mon, 31 May 2021 14:11:38 +0300 Subject: [PATCH 148/173] fix review comments Signed-off-by: Stepan Blyschak --- config/config_mgmt.py | 21 +++++++------- sonic_package_manager/manifest.py | 1 - sonic_package_manager/metadata.py | 6 ++-- .../service_creator/creator.py | 28 ++++++++++++------- .../test_service_creator.py | 6 ++-- 5 files changed, 35 insertions(+), 27 deletions(-) diff --git a/config/config_mgmt.py b/config/config_mgmt.py index 3139384916..4e34a7ae00 100644 --- a/config/config_mgmt.py +++ b/config/config_mgmt.py @@ -220,23 +220,23 @@ def writeConfigDB(self, jDiff): return - def add_module(self, yang_module_text, replace_if_exists=False): + def add_module(self, yang_module_str, replace_if_exists=False): """ Validate and add new YANG module to the system. Parameters: - yang_module_text (str): YANG module string. + yang_module_str (str): YANG module in string representation. Returns: None """ - module_name = self.get_module_name(yang_module_text) + module_name = self.get_module_name(yang_module_str) module_path = os.path.join(YANG_DIR, '{}.yang'.format(module_name)) if os.path.exists(module_path) and not replace_if_exists: raise Exception('{} already exists'.format(module_name)) with open(module_path, 'w') as module_file: - module_file.write(yang_module_text) + module_file.write(yang_module_str) try: self.__init_sonic_yang() except Exception: @@ -258,28 +258,29 @@ def remove_module(self, module_name): if not os.path.exists(module_path): return with open(module_path, 'r') as module_file: - yang_module_text = module_file.read() + yang_module_str = module_file.read() try: os.remove(module_path) self.__init_sonic_yang() except Exception: - self.add_module(yang_module_text) + self.add_module(yang_module_str) raise @staticmethod - def get_module_name(yang_module_text): + def get_module_name(yang_module_str): """ - Read yangs module name from yang_module_text + Read yangs module name from yang_module_str Parameters: - yang_module_text(str): YANG module string. + yang_module_str(str): YANG module string. Returns: str: Module name """ + # Instantiate new context since parse_module_mem() loads the module into context. sy = sonic_yang.SonicYang(YANG_DIR) - module = sy.ctx.parse_module_mem(yang_module_text, ly.LYS_IN_YANG) + module = sy.ctx.parse_module_mem(yang_module_str, ly.LYS_IN_YANG) return module.name() diff --git a/sonic_package_manager/manifest.py b/sonic_package_manager/manifest.py index b234d26ee8..216baef756 100644 --- a/sonic_package_manager/manifest.py +++ b/sonic_package_manager/manifest.py @@ -208,7 +208,6 @@ def unmarshal(self, value): ManifestField('clear', DefaultMarshaller(str), ''), ManifestField('auto-generate-show', DefaultMarshaller(bool), False), ManifestField('auto-generate-config', DefaultMarshaller(bool), False), - ManifestField('auto-generate-clear', DefaultMarshaller(bool), False), ]) ]) diff --git a/sonic_package_manager/metadata.py b/sonic_package_manager/metadata.py index 3475cf2c70..dc718375ed 100644 --- a/sonic_package_manager/metadata.py +++ b/sonic_package_manager/metadata.py @@ -73,7 +73,7 @@ class Metadata: manifest: Manifest components: Dict[str, Version] = field(default_factory=dict) - yang_module_text: Optional[str] = None + yang_module_str: Optional[str] = None class MetadataResolver: @@ -183,6 +183,6 @@ def from_labels(cls, labels: Dict[str, str]) -> Metadata: except ValueError as err: raise MetadataError(f'Failed to parse component version: {err}') - yang_module_text = sonic_metadata.get('yang-module') + yang_module_str = sonic_metadata.get('yang-module') - return Metadata(Manifest.marshal(manifest_dict), components, yang_module_text) + return Metadata(Manifest.marshal(manifest_dict), components, yang_module_str) diff --git a/sonic_package_manager/service_creator/creator.py b/sonic_package_manager/service_creator/creator.py index 3b1464ffa2..c81f5cddc3 100644 --- a/sonic_package_manager/service_creator/creator.py +++ b/sonic_package_manager/service_creator/creator.py @@ -505,10 +505,10 @@ def remove_config(self, package): None """ - if not package.metadata.yang_module_text: + if not package.metadata.yang_module_str: return - module_name = self.cfg_mgmt.get_module_name(package.metadata.yang_module_text) + module_name = self.cfg_mgmt.get_module_name(package.metadata.yang_module_str) for tablename, module in self.cfg_mgmt.sy.confDbYangMap.items(): if module.get('module') != module_name: continue @@ -532,6 +532,10 @@ def validate_config(self, config): config = sonic_cfggen.FormatConverter.to_serialized(config) log.debug(f'validating configuration {pformat(config)}') # This will raise exception if configuration is not valid. + # NOTE: loadData() modifies the state of ConfigMgmt instance. + # This is not desired for configuration validation only purpose. + # Although the config loaded into ConfigMgmt instance is not + # interesting in this application so we don't care. self.cfg_mgmt.loadData(config) def install_yang_module(self, package: Package): @@ -543,10 +547,10 @@ def install_yang_module(self, package: Package): None """ - if not package.metadata.yang_module_text: + if not package.metadata.yang_module_str: return - self.cfg_mgmt.add_module(package.metadata.yang_module_text) + self.cfg_mgmt.add_module(package.metadata.yang_module_str) def uninstall_yang_module(self, package: Package): """ Uninstall package's yang module in the system. @@ -557,10 +561,10 @@ def uninstall_yang_module(self, package: Package): None """ - if not package.metadata.yang_module_text: + if not package.metadata.yang_module_str: return - module_name = self.cfg_mgmt.get_module_name(package.metadata.yang_module_text) + module_name = self.cfg_mgmt.get_module_name(package.metadata.yang_module_str) self.cfg_mgmt.remove_module(module_name) def install_autogen_cli_all(self, package: Package): @@ -597,11 +601,13 @@ def install_autogen_cli(self, package: Package, command: str): None """ - if package.metadata.yang_module_text is None: + if package.metadata.yang_module_str is None: + return + if f'auto-generate-{command}' not in package.manifest['cli']: return if not package.manifest['cli'][f'auto-generate-{command}']: return - module_name = self.cfg_mgmt.get_module_name(package.metadata.yang_module_text) + module_name = self.cfg_mgmt.get_module_name(package.metadata.yang_module_str) self.cli_gen.generate_cli_plugin(command, module_name) log.debug(f'{command} command line interface autogenerated for {module_name}') @@ -615,11 +621,13 @@ def uninstall_autogen_cli(self, package: Package, command: str): None """ - if package.metadata.yang_module_text is None: + if package.metadata.yang_module_str is None: + return + if f'auto-generate-{command}' not in package.manifest['cli']: return if not package.manifest['cli'][f'auto-generate-{command}']: return - module_name = self.cfg_mgmt.get_module_name(package.metadata.yang_module_text) + module_name = self.cfg_mgmt.get_module_name(package.metadata.yang_module_str) self.cli_gen.remove_cli_plugin(command, module_name) log.debug(f'{command} command line interface removed for {module_name}') diff --git a/tests/sonic_package_manager/test_service_creator.py b/tests/sonic_package_manager/test_service_creator.py index be92171b3f..2951196a2d 100644 --- a/tests/sonic_package_manager/test_service_creator.py +++ b/tests/sonic_package_manager/test_service_creator.py @@ -135,7 +135,7 @@ def test_service_creator_yang(sonic_fs, manifest, mock_sonic_db, }) entry = PackageEntry('test', 'azure/sonic-test') - package = Package(entry, Metadata(manifest, yang_module_text=test_yang)) + package = Package(entry, Metadata(manifest, yang_module_str=test_yang)) service_creator.create(package) mock_config_mgmt.add_module.assert_called_with(test_yang) @@ -149,7 +149,7 @@ def test_service_creator_yang(sonic_fs, manifest, mock_sonic_db, }, }, } - package = Package(entry, Metadata(manifest, yang_module_text=test_yang)) + package = Package(entry, Metadata(manifest, yang_module_str=test_yang)) service_creator.create(package) @@ -184,7 +184,7 @@ def test_service_creator_autocli(sonic_fs, manifest, mock_cli_gen, manifest['cli']['auto-generate-config'] = True entry = PackageEntry('test', 'azure/sonic-test') - package = Package(entry, Metadata(manifest, yang_module_text=test_yang)) + package = Package(entry, Metadata(manifest, yang_module_str=test_yang)) mock_config_mgmt.get_module_name = Mock(return_value=test_yang_module) service_creator.create(package) From 8424b0bbdad5ddcfee04cfd96224d58a4101aae2 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Mon, 31 May 2021 14:20:08 +0300 Subject: [PATCH 149/173] fix review comments Signed-off-by: Stepan Blyschak --- sonic_package_manager/manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sonic_package_manager/manager.py b/sonic_package_manager/manager.py index 571a73a00f..5fe48c5f0f 100644 --- a/sonic_package_manager/manager.py +++ b/sonic_package_manager/manager.py @@ -995,7 +995,7 @@ def get_manager() -> 'PackageManager': registry_resolver = RegistryResolver() metadata_resolver = MetadataResolver(docker_api, registry_resolver) cfg_mgmt = config_mgmt.ConfigMgmt() - cli_generator = CliGenerator() + cli_generator = CliGenerator(log) feature_registry = FeatureRegistry(SonicDB) service_creator = ServiceCreator(feature_registry, SonicDB, From fab17a9fc9b64bcfff354f4b361d69ffd4f5dbc3 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Tue, 1 Jun 2021 13:48:50 +0300 Subject: [PATCH 150/173] fix review comments Signed-off-by: Stepan Blyschak --- sonic_package_manager/service_creator/sonic_db.py | 10 +++++----- tests/sonic_package_manager/test_service_creator.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/sonic_package_manager/service_creator/sonic_db.py b/sonic_package_manager/service_creator/sonic_db.py index 36d7b4a744..6b617cb802 100644 --- a/sonic_package_manager/service_creator/sonic_db.py +++ b/sonic_package_manager/service_creator/sonic_db.py @@ -17,7 +17,9 @@ class PersistentConfigDbConnector: - """ swsscommon.ConfigDBConnector adapter for persistent DBs. """ + """ This class implements swsscommon.ConfigDBConnector methods for persistent DBs (JSON files). + For method description refer to swsscommon.ConfigDBConnector. + """ def __init__(self, filepath): self._filepath = filepath @@ -132,12 +134,10 @@ def get_persistent_db_connector(cls): if not os.path.exists(CONFIG_DB_JSON): return None - conn = PersistentConfigDbConnector(CONFIG_DB_JSON) - return conn + return PersistentConfigDbConnector(CONFIG_DB_JSON) @classmethod def get_initial_db_connector(cls): """ Returns initial DB connector. """ - conn = PersistentConfigDbConnector(INIT_CFG_JSON) - return conn + return PersistentConfigDbConnector(INIT_CFG_JSON) diff --git a/tests/sonic_package_manager/test_service_creator.py b/tests/sonic_package_manager/test_service_creator.py index 2951196a2d..456cc71a4a 100644 --- a/tests/sonic_package_manager/test_service_creator.py +++ b/tests/sonic_package_manager/test_service_creator.py @@ -153,7 +153,7 @@ def test_service_creator_yang(sonic_fs, manifest, mock_sonic_db, service_creator.create(package) - mock_config_mgmt.add_module.assert_called_with('TEST YANG') + mock_config_mgmt.add_module.assert_called_with(test_yang) mock_connector.mod_config.assert_called_with( { From 0aa7df7fa74a91e3372cd0ee0cc5c86efb3a996e Mon Sep 17 00:00:00 2001 From: Stepan Blyshchak Date: Thu, 8 Jul 2021 14:42:15 +0300 Subject: [PATCH 151/173] fix LGTM warning Signed-off-by: Stepan Blyshchak --- sonic_package_manager/manager.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sonic_package_manager/manager.py b/sonic_package_manager/manager.py index 5fe48c5f0f..2791bf1939 100644 --- a/sonic_package_manager/manager.py +++ b/sonic_package_manager/manager.py @@ -528,7 +528,6 @@ def upgrade_from_source(self, ) old_feature = old_package.manifest['service']['name'] - new_feature = new_package.manifest['service']['name'] old_version = old_package.manifest['package']['version'] new_version = new_package.manifest['package']['version'] From 4345a6a815bab92c51cfdafc39f50e93cd0b692b Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Tue, 20 Jul 2021 16:06:33 +0300 Subject: [PATCH 152/173] initialize ConfigMgmt with init_cfg.json instead of running config to allow running in chroot environment Signed-off-by: Stepan Blyschak --- sonic_package_manager/manager.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/sonic_package_manager/manager.py b/sonic_package_manager/manager.py index 2791bf1939..836a992f0a 100644 --- a/sonic_package_manager/manager.py +++ b/sonic_package_manager/manager.py @@ -48,7 +48,10 @@ run_command ) from sonic_package_manager.service_creator.feature import FeatureRegistry -from sonic_package_manager.service_creator.sonic_db import SonicDB +from sonic_package_manager.service_creator.sonic_db import ( + INIT_CFG_JSON, + SonicDB +) from sonic_package_manager.service_creator.utils import in_chroot from sonic_package_manager.source import ( PackageSource, @@ -993,7 +996,7 @@ def get_manager() -> 'PackageManager': docker_api = DockerApi(docker.from_env(), ProgressManager()) registry_resolver = RegistryResolver() metadata_resolver = MetadataResolver(docker_api, registry_resolver) - cfg_mgmt = config_mgmt.ConfigMgmt() + cfg_mgmt = config_mgmt.ConfigMgmt(source=INIT_CFG_JSON) cli_generator = CliGenerator(log) feature_registry = FeatureRegistry(SonicDB) service_creator = ServiceCreator(feature_registry, From 94cde8e8117f5830ed20ae04aaa4a9b5098bd2f1 Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Mon, 30 Aug 2021 11:36:19 +0000 Subject: [PATCH 153/173] Changed tabs to spaces Signed-off-by: Vadym Hlushko --- .../cli_autogen_input/assert_dictionaries.py | 3 +- tests/cli_autogen_input/sonic-1-list.yang | 36 ++-- .../sonic-1-object-container.yang | 28 +-- .../sonic-1-table-container.yang | 20 +- tests/cli_autogen_input/sonic-2-lists.yang | 52 ++--- .../sonic-2-object-containers.yang | 36 ++-- .../sonic-2-table-containers.yang | 26 +-- .../sonic-choice-complex.yang | 174 ++++++++--------- .../sonic-dynamic-object-complex-1.yang | 106 +++++----- .../sonic-dynamic-object-complex-2.yang | 160 +++++++-------- tests/cli_autogen_input/sonic-grouping-1.yang | 32 +-- tests/cli_autogen_input/sonic-grouping-2.yang | 32 +-- .../sonic-grouping-complex.yang | 184 +++++++++--------- .../sonic-static-object-complex-1.yang | 90 ++++----- .../sonic-static-object-complex-2.yang | 114 +++++------ 15 files changed, 547 insertions(+), 546 deletions(-) diff --git a/tests/cli_autogen_input/assert_dictionaries.py b/tests/cli_autogen_input/assert_dictionaries.py index 263e48366d..bed2a4a06a 100644 --- a/tests/cli_autogen_input/assert_dictionaries.py +++ b/tests/cli_autogen_input/assert_dictionaries.py @@ -622,4 +622,5 @@ ] } ] -} \ No newline at end of file +} + diff --git a/tests/cli_autogen_input/sonic-1-list.yang b/tests/cli_autogen_input/sonic-1-list.yang index 79a6529b3d..bc8603add4 100644 --- a/tests/cli_autogen_input/sonic-1-list.yang +++ b/tests/cli_autogen_input/sonic-1-list.yang @@ -2,28 +2,28 @@ module sonic-1-list { yang-version 1.1; - namespace "http://github.com/Azure/s-1-list"; - prefix s-1-list; + namespace "http://github.com/Azure/s-1-list"; + prefix s-1-list; - container sonic-1-list { - /* sonic-1-list - top level container */ + container sonic-1-list { + /* sonic-1-list - top level container */ - container TABLE_1 { - /* TABLE_1 - table container */ + container TABLE_1 { + /* TABLE_1 - table container */ - description "TABLE_1 description"; + description "TABLE_1 description"; - list TABLE_1_LIST { - /* TABLE_1 - object container */ + list TABLE_1_LIST { + /* TABLE_1 - object container */ - description "TABLE_1_LIST description"; + description "TABLE_1_LIST description"; - key "key_name"; + key "key_name"; - leaf key_name { - type string; - } - } - } - } -} \ No newline at end of file + leaf key_name { + type string; + } + } + } + } +} diff --git a/tests/cli_autogen_input/sonic-1-object-container.yang b/tests/cli_autogen_input/sonic-1-object-container.yang index e28ef7f90a..8d19979157 100644 --- a/tests/cli_autogen_input/sonic-1-object-container.yang +++ b/tests/cli_autogen_input/sonic-1-object-container.yang @@ -2,22 +2,22 @@ module sonic-1-object-container { yang-version 1.1; - namespace "http://github.com/Azure/s-1-object"; - prefix s-1-object; + namespace "http://github.com/Azure/s-1-object"; + prefix s-1-object; - container sonic-1-object-container { - /* sonic-1-object-container - top level container */ + container sonic-1-object-container { + /* sonic-1-object-container - top level container */ - container TABLE_1 { - /* TABLE_1 - table container */ + container TABLE_1 { + /* TABLE_1 - table container */ - description "TABLE_1 description"; + description "TABLE_1 description"; - container OBJECT_1 { - /* OBJECT_1 - object container */ + container OBJECT_1 { + /* OBJECT_1 - object container */ - description "OBJECT_1 description"; - } - } - } -} \ No newline at end of file + description "OBJECT_1 description"; + } + } + } +} diff --git a/tests/cli_autogen_input/sonic-1-table-container.yang b/tests/cli_autogen_input/sonic-1-table-container.yang index 58e7293c0d..36b98415e5 100644 --- a/tests/cli_autogen_input/sonic-1-table-container.yang +++ b/tests/cli_autogen_input/sonic-1-table-container.yang @@ -2,16 +2,16 @@ module sonic-1-table-container { yang-version 1.1; - namespace "http://github.com/Azure/s-1-table"; - prefix s-1-table; + namespace "http://github.com/Azure/s-1-table"; + prefix s-1-table; - container sonic-1-table-container { - /* sonic-1-table-container - top level container */ + container sonic-1-table-container { + /* sonic-1-table-container - top level container */ - container TABLE_1 { - /* TABLE_1 - table container */ + container TABLE_1 { + /* TABLE_1 - table container */ - description "TABLE_1 description"; - } - } -} \ No newline at end of file + description "TABLE_1 description"; + } + } +} diff --git a/tests/cli_autogen_input/sonic-2-lists.yang b/tests/cli_autogen_input/sonic-2-lists.yang index b20200415b..fce9704f00 100644 --- a/tests/cli_autogen_input/sonic-2-lists.yang +++ b/tests/cli_autogen_input/sonic-2-lists.yang @@ -2,41 +2,41 @@ module sonic-2-lists { yang-version 1.1; - namespace "http://github.com/Azure/s-2-lists"; - prefix s-2-lists; + namespace "http://github.com/Azure/s-2-lists"; + prefix s-2-lists; - container sonic-2-lists { - /* sonic-2-lists - top level container */ + container sonic-2-lists { + /* sonic-2-lists - top level container */ - container TABLE_1 { - /* TALBE_1 - table container */ + container TABLE_1 { + /* TALBE_1 - table container */ - description "TABLE_1 description"; + description "TABLE_1 description"; - list TABLE_1_LIST_1 { - /* TALBE_1_LIST_1 - object container */ + list TABLE_1_LIST_1 { + /* TALBE_1_LIST_1 - object container */ - description "TABLE_1_LIST_1 description"; + description "TABLE_1_LIST_1 description"; - key "key_name1"; + key "key_name1"; - leaf key_name1 { - type string; - } - } + leaf key_name1 { + type string; + } + } - list TABLE_1_LIST_2 { - /* TALBE_1_LIST_2 - object container */ + list TABLE_1_LIST_2 { + /* TALBE_1_LIST_2 - object container */ - description "TABLE_1_LIST_2 description"; + description "TABLE_1_LIST_2 description"; - key "key_name2"; + key "key_name2"; - leaf key_name2 { - type string; - } - } - } - } -} \ No newline at end of file + leaf key_name2 { + type string; + } + } + } + } +} diff --git a/tests/cli_autogen_input/sonic-2-object-containers.yang b/tests/cli_autogen_input/sonic-2-object-containers.yang index 249faf4c89..e633b66246 100644 --- a/tests/cli_autogen_input/sonic-2-object-containers.yang +++ b/tests/cli_autogen_input/sonic-2-object-containers.yang @@ -2,28 +2,28 @@ module sonic-2-object-containers { yang-version 1.1; - namespace "http://github.com/Azure/s-2-object"; - prefix s-2-object; + namespace "http://github.com/Azure/s-2-object"; + prefix s-2-object; - container sonic-2-object-containers { - /* sonic-2-object-containers - top level container */ + container sonic-2-object-containers { + /* sonic-2-object-containers - top level container */ - container TABLE_1 { - /* TABLE_1 - table container */ + container TABLE_1 { + /* TABLE_1 - table container */ - description "FIRST_TABLE description"; + description "FIRST_TABLE description"; - container OBJECT_1 { - /* OBJECT_1 - object container */ + container OBJECT_1 { + /* OBJECT_1 - object container */ - description "OBJECT_1 description"; - } + description "OBJECT_1 description"; + } - container OBJECT_2 { - /* OBJECT_2 - object container */ + container OBJECT_2 { + /* OBJECT_2 - object container */ - description "OBJECT_2 description"; - } - } - } -} \ No newline at end of file + description "OBJECT_2 description"; + } + } + } +} diff --git a/tests/cli_autogen_input/sonic-2-table-containers.yang b/tests/cli_autogen_input/sonic-2-table-containers.yang index 393512a313..f5284c67ee 100644 --- a/tests/cli_autogen_input/sonic-2-table-containers.yang +++ b/tests/cli_autogen_input/sonic-2-table-containers.yang @@ -2,22 +2,22 @@ module sonic-2-table-containers { yang-version 1.1; - namespace "http://github.com/Azure/s-2-table"; - prefix s-2-table; + namespace "http://github.com/Azure/s-2-table"; + prefix s-2-table; - container sonic-2-table-containers { - /* sonic-2-table-containers - top level container */ + container sonic-2-table-containers { + /* sonic-2-table-containers - top level container */ - container TABLE_1 { - /* TABLE_1 - table container */ + container TABLE_1 { + /* TABLE_1 - table container */ - description "TABLE_1 description"; - } + description "TABLE_1 description"; + } - container TABLE_2 { - /* TABLE_2 - table container */ + container TABLE_2 { + /* TABLE_2 - table container */ - description "TABLE_2 description"; - } - } + description "TABLE_2 description"; + } + } } diff --git a/tests/cli_autogen_input/sonic-choice-complex.yang b/tests/cli_autogen_input/sonic-choice-complex.yang index 7d6a66d89f..9d6e0de9ee 100644 --- a/tests/cli_autogen_input/sonic-choice-complex.yang +++ b/tests/cli_autogen_input/sonic-choice-complex.yang @@ -2,90 +2,90 @@ module sonic-choice-complex { yang-version 1.1; - namespace "http://github.com/Azure/choice-complex"; - prefix choice-complex; - - import sonic-grouping-1 { - prefix sgroup1; - } - - import sonic-grouping-2 { - prefix sgroup2; - } - - grouping GR_5 { - leaf GR_5_LEAF_1 { - type string; - } - - leaf GR_5_LEAF_2 { - type string; - } - } - - grouping GR_6 { - leaf GR_6_LEAF_1 { - type string; - } - - leaf GR_6_LEAF_2 { - type string; - } - } - - container sonic-choice-complex { - /* sonic-choice-complex - top level container */ - - container TABLE_1 { - /* TABLE_1 - table container */ - - description "TABLE_1 description"; - - container OBJECT_1 { - /* OBJECT_1 - object container, it have - * 1 choice, which have 2 cases. - * first case have: 1 leaf, 1 leaf-list, 1 uses - * second case have: 2 leafs, 2 leaf-lists, 2 uses - */ - - description "OBJECT_1 description"; - - choice CHOICE_1 { - case CHOICE_1_CASE_1 { - leaf LEAF_1 { - type uint16; - } - - leaf-list LEAF_LIST_1 { - type string; - } - - uses sgroup1:GR_1; - } - - case CHOICE_1_CASE_2 { - leaf LEAF_2 { - type string; - } - - leaf LEAF_3 { - type string; - } - - leaf-list LEAF_LIST_2 { - type string; - } - - leaf-list LEAF_LIST_3 { - type string; - } - - uses GR_5; - uses sgroup1:GR_2; - uses sgroup2:GR_3; - } - } - } - } - } -} \ No newline at end of file + namespace "http://github.com/Azure/choice-complex"; + prefix choice-complex; + + import sonic-grouping-1 { + prefix sgroup1; + } + + import sonic-grouping-2 { + prefix sgroup2; + } + + grouping GR_5 { + leaf GR_5_LEAF_1 { + type string; + } + + leaf GR_5_LEAF_2 { + type string; + } + } + + grouping GR_6 { + leaf GR_6_LEAF_1 { + type string; + } + + leaf GR_6_LEAF_2 { + type string; + } + } + + container sonic-choice-complex { + /* sonic-choice-complex - top level container */ + + container TABLE_1 { + /* TABLE_1 - table container */ + + description "TABLE_1 description"; + + container OBJECT_1 { + /* OBJECT_1 - object container, it have + * 1 choice, which have 2 cases. + * first case have: 1 leaf, 1 leaf-list, 1 uses + * second case have: 2 leafs, 2 leaf-lists, 2 uses + */ + + description "OBJECT_1 description"; + + choice CHOICE_1 { + case CHOICE_1_CASE_1 { + leaf LEAF_1 { + type uint16; + } + + leaf-list LEAF_LIST_1 { + type string; + } + + uses sgroup1:GR_1; + } + + case CHOICE_1_CASE_2 { + leaf LEAF_2 { + type string; + } + + leaf LEAF_3 { + type string; + } + + leaf-list LEAF_LIST_2 { + type string; + } + + leaf-list LEAF_LIST_3 { + type string; + } + + uses GR_5; + uses sgroup1:GR_2; + uses sgroup2:GR_3; + } + } + } + } + } +} diff --git a/tests/cli_autogen_input/sonic-dynamic-object-complex-1.yang b/tests/cli_autogen_input/sonic-dynamic-object-complex-1.yang index 9beb98549d..383e94fb43 100644 --- a/tests/cli_autogen_input/sonic-dynamic-object-complex-1.yang +++ b/tests/cli_autogen_input/sonic-dynamic-object-complex-1.yang @@ -2,56 +2,56 @@ module sonic-dynamic-object-complex-1 { yang-version 1.1; - namespace "http://github.com/Azure/dynamic-complex-1"; - prefix dynamic-complex-1; - - container sonic-dynamic-object-complex-1 { - /* sonic-dynamic-object-complex-1 - top level container */ - - container TABLE_1 { - /* TABLE_1 - table container */ - - description "TABLE_1 description"; - - list OBJECT_1_LIST { - /* OBJECT_1_LIST - dynamic object container, it have: - * 1 key, - * 1 leaf, - * 1 leaf-list - * 1 choice - */ - - description "OBJECT_1_LIST description"; - - key "KEY_LEAF_1"; - - leaf KEY_LEAF_1 { - description "KEY_LEAF_1 description"; - type string; - } - - leaf OBJ_1_LEAF_1 { - description "OBJ_1_LEAF_1 description"; - type string; - } - - leaf-list OBJ_1_LEAF_LIST_1 { - type string; - } - - choice OBJ_1_CHOICE_1 { - case OBJ_1_CHOICE_1_CASE_1 { - leaf OBJ_1_CHOICE_1_LEAF_1 { - type uint16; - } - } - case OBJ_1_CHOICE_1_CASE_2 { - leaf OBJ_1_CHOICE_1_LEAF_2 { - type string; - } - } - } - } - } - } -} \ No newline at end of file + namespace "http://github.com/Azure/dynamic-complex-1"; + prefix dynamic-complex-1; + + container sonic-dynamic-object-complex-1 { + /* sonic-dynamic-object-complex-1 - top level container */ + + container TABLE_1 { + /* TABLE_1 - table container */ + + description "TABLE_1 description"; + + list OBJECT_1_LIST { + /* OBJECT_1_LIST - dynamic object container, it have: + * 1 key, + * 1 leaf, + * 1 leaf-list + * 1 choice + */ + + description "OBJECT_1_LIST description"; + + key "KEY_LEAF_1"; + + leaf KEY_LEAF_1 { + description "KEY_LEAF_1 description"; + type string; + } + + leaf OBJ_1_LEAF_1 { + description "OBJ_1_LEAF_1 description"; + type string; + } + + leaf-list OBJ_1_LEAF_LIST_1 { + type string; + } + + choice OBJ_1_CHOICE_1 { + case OBJ_1_CHOICE_1_CASE_1 { + leaf OBJ_1_CHOICE_1_LEAF_1 { + type uint16; + } + } + case OBJ_1_CHOICE_1_CASE_2 { + leaf OBJ_1_CHOICE_1_LEAF_2 { + type string; + } + } + } + } + } + } +} diff --git a/tests/cli_autogen_input/sonic-dynamic-object-complex-2.yang b/tests/cli_autogen_input/sonic-dynamic-object-complex-2.yang index 00e25c8135..a365b014ad 100644 --- a/tests/cli_autogen_input/sonic-dynamic-object-complex-2.yang +++ b/tests/cli_autogen_input/sonic-dynamic-object-complex-2.yang @@ -2,83 +2,83 @@ module sonic-dynamic-object-complex-2 { yang-version 1.1; - namespace "http://github.com/Azure/dynamic-complex-2"; - prefix dynamic-complex-2; - - container sonic-dynamic-object-complex-2 { - /* sonic-dynamic-object-complex-2 - top level container */ - - container TABLE_1 { - /* TABLE_1 - table container */ - - description "TABLE_1 description"; - - list OBJECT_1_LIST { - /* OBJECT_1_LIST - dynamic object container, it have: - * 2 keys - * 2 leaf, - * 2 leaf-list - * 2 choice - */ - - description "OBJECT_1_LIST description"; - - key "KEY_LEAF_1 KEY_LEAF_2"; - - leaf KEY_LEAF_1 { - description "KEY_LEAF_1 description"; - type string; - } - - leaf KEY_LEAF_2 { - description "KEY_LEAF_2 description"; - type string; - } - - leaf OBJ_1_LEAF_1 { - description "OBJ_1_LEAF_1 description"; - type string; - } - - leaf OBJ_1_LEAF_2 { - description "OBJ_1_LEAF_2 description"; - type string; - } - - leaf-list OBJ_1_LEAF_LIST_1 { - type string; - } - - leaf-list OBJ_1_LEAF_LIST_2 { - type string; - } - - choice OBJ_1_CHOICE_1 { - case OBJ_1_CHOICE_1_CASE_1 { - leaf OBJ_1_CHOICE_1_LEAF_1 { - type uint16; - } - } - case OBJ_1_CHOICE_1_CASE_2 { - leaf OBJ_1_CHOICE_1_LEAF_2 { - type string; - } - } - } - - choice OBJ_1_CHOICE_2 { - case OBJ_1_CHOICE_2_CASE_1 { - leaf OBJ_1_CHOICE_2_LEAF_1 { - type uint16; - } - } - case OBJ_1_CHOICE_2_CASE_2 { - leaf OBJ_1_CHOICE_2_LEAF_2 { - type string; - } - } - } - } - } - } -} \ No newline at end of file + namespace "http://github.com/Azure/dynamic-complex-2"; + prefix dynamic-complex-2; + + container sonic-dynamic-object-complex-2 { + /* sonic-dynamic-object-complex-2 - top level container */ + + container TABLE_1 { + /* TABLE_1 - table container */ + + description "TABLE_1 description"; + + list OBJECT_1_LIST { + /* OBJECT_1_LIST - dynamic object container, it have: + * 2 keys + * 2 leaf, + * 2 leaf-list + * 2 choice + */ + + description "OBJECT_1_LIST description"; + + key "KEY_LEAF_1 KEY_LEAF_2"; + + leaf KEY_LEAF_1 { + description "KEY_LEAF_1 description"; + type string; + } + + leaf KEY_LEAF_2 { + description "KEY_LEAF_2 description"; + type string; + } + + leaf OBJ_1_LEAF_1 { + description "OBJ_1_LEAF_1 description"; + type string; + } + + leaf OBJ_1_LEAF_2 { + description "OBJ_1_LEAF_2 description"; + type string; + } + + leaf-list OBJ_1_LEAF_LIST_1 { + type string; + } + + leaf-list OBJ_1_LEAF_LIST_2 { + type string; + } + + choice OBJ_1_CHOICE_1 { + case OBJ_1_CHOICE_1_CASE_1 { + leaf OBJ_1_CHOICE_1_LEAF_1 { + type uint16; + } + } + case OBJ_1_CHOICE_1_CASE_2 { + leaf OBJ_1_CHOICE_1_LEAF_2 { + type string; + } + } + } + + choice OBJ_1_CHOICE_2 { + case OBJ_1_CHOICE_2_CASE_1 { + leaf OBJ_1_CHOICE_2_LEAF_1 { + type uint16; + } + } + case OBJ_1_CHOICE_2_CASE_2 { + leaf OBJ_1_CHOICE_2_LEAF_2 { + type string; + } + } + } + } + } + } +} diff --git a/tests/cli_autogen_input/sonic-grouping-1.yang b/tests/cli_autogen_input/sonic-grouping-1.yang index 831c3a4ad8..bf0be792f5 100644 --- a/tests/cli_autogen_input/sonic-grouping-1.yang +++ b/tests/cli_autogen_input/sonic-grouping-1.yang @@ -2,24 +2,24 @@ module sonic-grouping-1{ yang-version 1.1; - namespace "http://github.com/Azure/s-grouping-1"; - prefix s-grouping-1; + namespace "http://github.com/Azure/s-grouping-1"; + prefix s-grouping-1; - grouping GR_1 { - leaf GR_1_LEAF_1 { - type string; - } - leaf GR_1_LEAF_2 { - type string; - } + grouping GR_1 { + leaf GR_1_LEAF_1 { + type string; + } + leaf GR_1_LEAF_2 { + type string; + } } - grouping GR_2 { - leaf GR_2_LEAF_1 { - type string; - } - leaf GR_2_LEAF_2 { - type string; + grouping GR_2 { + leaf GR_2_LEAF_1 { + type string; + } + leaf GR_2_LEAF_2 { + type string; } } -} \ No newline at end of file +} diff --git a/tests/cli_autogen_input/sonic-grouping-2.yang b/tests/cli_autogen_input/sonic-grouping-2.yang index bfaa13db15..58e9df6621 100644 --- a/tests/cli_autogen_input/sonic-grouping-2.yang +++ b/tests/cli_autogen_input/sonic-grouping-2.yang @@ -2,24 +2,24 @@ module sonic-grouping-2 { yang-version 1.1; - namespace "http://github.com/Azure/s-grouping-2"; - prefix s-grouping-2; + namespace "http://github.com/Azure/s-grouping-2"; + prefix s-grouping-2; - grouping GR_3 { - leaf GR_3_LEAF_1 { - type string; - } - leaf GR_3_LEAF_2 { - type string; - } + grouping GR_3 { + leaf GR_3_LEAF_1 { + type string; + } + leaf GR_3_LEAF_2 { + type string; + } } - grouping GR_4 { - leaf GR_4_LEAF_1 { - type string; - } - leaf GR_4_LEAF_2 { - type string; + grouping GR_4 { + leaf GR_4_LEAF_1 { + type string; + } + leaf GR_4_LEAF_2 { + type string; } } -} \ No newline at end of file +} diff --git a/tests/cli_autogen_input/sonic-grouping-complex.yang b/tests/cli_autogen_input/sonic-grouping-complex.yang index d6ed68563a..22956789b0 100644 --- a/tests/cli_autogen_input/sonic-grouping-complex.yang +++ b/tests/cli_autogen_input/sonic-grouping-complex.yang @@ -2,95 +2,95 @@ module sonic-grouping-complex { yang-version 1.1; - namespace "http://github.com/Azure/grouping-complex"; - prefix grouping-complex; - - import sonic-grouping-1 { - prefix sgroup1; - } - - import sonic-grouping-2 { - prefix sgroup2; - } - - grouping GR_5 { - leaf GR_5_LEAF_1 { - type string; - } - - leaf-list GR_5_LEAF_LIST_1 { - type string; - } - } - - grouping GR_6 { - leaf GR_6_LEAF_1 { - type string; - } - - leaf GR_6_LEAF_2 { - type string; - } - - choice GR_6_CHOICE_1 { - case CHOICE_1_CASE_1 { - leaf GR_6_CASE_1_LEAF_1 { - type uint16; - } - - leaf-list GR_6_CASE_1_LEAF_LIST_1 { - type string; - } - } - - case CHOICE_1_CASE_2 { - leaf GR_6_CASE_2_LEAF_1 { - type uint16; - } - - leaf GR_6_CASE_2_LEAF_2 { - type uint16; - } - - leaf-list GR_6_CASE_2_LEAF_LIST_1 { - type string; - } - - leaf-list GR_6_CASE_2_LEAF_LIST_2 { - type string; - } - } - } - } - - container sonic-grouping-complex { - /* sonic-grouping-complex - top level container */ - - container TABLE_1 { - /* TABLE_1 - table container */ - - description "TABLE_1 description"; - - container OBJECT_1 { - /* OBJECT_1 - object container, it have - * 1 choice, which have 2 cases. - * first case have: 1 leaf, 1 leaf-list, 1 uses - * second case have: 2 leafs, 2 leaf-lists, 2 uses - */ - - description "OBJECT_1 description"; - - uses sgroup1:GR_1; - } - - container OBJECT_2 { - - description "OBJECT_2 description"; - - uses GR_5; - uses GR_6; - uses sgroup2:GR_4; - } - } - } -} \ No newline at end of file + namespace "http://github.com/Azure/grouping-complex"; + prefix grouping-complex; + + import sonic-grouping-1 { + prefix sgroup1; + } + + import sonic-grouping-2 { + prefix sgroup2; + } + + grouping GR_5 { + leaf GR_5_LEAF_1 { + type string; + } + + leaf-list GR_5_LEAF_LIST_1 { + type string; + } + } + + grouping GR_6 { + leaf GR_6_LEAF_1 { + type string; + } + + leaf GR_6_LEAF_2 { + type string; + } + + choice GR_6_CHOICE_1 { + case CHOICE_1_CASE_1 { + leaf GR_6_CASE_1_LEAF_1 { + type uint16; + } + + leaf-list GR_6_CASE_1_LEAF_LIST_1 { + type string; + } + } + + case CHOICE_1_CASE_2 { + leaf GR_6_CASE_2_LEAF_1 { + type uint16; + } + + leaf GR_6_CASE_2_LEAF_2 { + type uint16; + } + + leaf-list GR_6_CASE_2_LEAF_LIST_1 { + type string; + } + + leaf-list GR_6_CASE_2_LEAF_LIST_2 { + type string; + } + } + } + } + + container sonic-grouping-complex { + /* sonic-grouping-complex - top level container */ + + container TABLE_1 { + /* TABLE_1 - table container */ + + description "TABLE_1 description"; + + container OBJECT_1 { + /* OBJECT_1 - object container, it have + * 1 choice, which have 2 cases. + * first case have: 1 leaf, 1 leaf-list, 1 uses + * second case have: 2 leafs, 2 leaf-lists, 2 uses + */ + + description "OBJECT_1 description"; + + uses sgroup1:GR_1; + } + + container OBJECT_2 { + + description "OBJECT_2 description"; + + uses GR_5; + uses GR_6; + uses sgroup2:GR_4; + } + } + } +} diff --git a/tests/cli_autogen_input/sonic-static-object-complex-1.yang b/tests/cli_autogen_input/sonic-static-object-complex-1.yang index a7dfee86ab..fa082d3b25 100644 --- a/tests/cli_autogen_input/sonic-static-object-complex-1.yang +++ b/tests/cli_autogen_input/sonic-static-object-complex-1.yang @@ -2,48 +2,48 @@ module sonic-static-object-complex-1 { yang-version 1.1; - namespace "http://github.com/Azure/static-complex-1"; - prefix static-complex-1; - - container sonic-static-object-complex-1 { - /* sonic-static-object-complex-1 - top level container */ - - container TABLE_1 { - /* TABLE_1 - table container */ - - description "TABLE_1 description"; - - container OBJECT_1 { - /* OBJECT_1 - object container, it have: - * 1 leaf, - * 1 leaf-list - * 1 choice - */ - - description "OBJECT_1 description"; - - leaf OBJ_1_LEAF_1 { - description "OBJ_1_LEAF_1 description"; - type string; - } - - leaf-list OBJ_1_LEAF_LIST_1 { - type string; - } - - choice OBJ_1_CHOICE_1 { - case OBJ_1_CHOICE_1_CASE_1 { - leaf OBJ_1_CHOICE_1_LEAF_1 { - type uint16; - } - } - case OBJ_1_CHOICE_1_CASE_2 { - leaf OBJ_1_CHOICE_1_LEAF_2 { - type string; - } - } - } - } - } - } -} \ No newline at end of file + namespace "http://github.com/Azure/static-complex-1"; + prefix static-complex-1; + + container sonic-static-object-complex-1 { + /* sonic-static-object-complex-1 - top level container */ + + container TABLE_1 { + /* TABLE_1 - table container */ + + description "TABLE_1 description"; + + container OBJECT_1 { + /* OBJECT_1 - object container, it have: + * 1 leaf, + * 1 leaf-list + * 1 choice + */ + + description "OBJECT_1 description"; + + leaf OBJ_1_LEAF_1 { + description "OBJ_1_LEAF_1 description"; + type string; + } + + leaf-list OBJ_1_LEAF_LIST_1 { + type string; + } + + choice OBJ_1_CHOICE_1 { + case OBJ_1_CHOICE_1_CASE_1 { + leaf OBJ_1_CHOICE_1_LEAF_1 { + type uint16; + } + } + case OBJ_1_CHOICE_1_CASE_2 { + leaf OBJ_1_CHOICE_1_LEAF_2 { + type string; + } + } + } + } + } + } +} diff --git a/tests/cli_autogen_input/sonic-static-object-complex-2.yang b/tests/cli_autogen_input/sonic-static-object-complex-2.yang index 451a445ce6..4e53b2e1b1 100644 --- a/tests/cli_autogen_input/sonic-static-object-complex-2.yang +++ b/tests/cli_autogen_input/sonic-static-object-complex-2.yang @@ -2,70 +2,70 @@ module sonic-static-object-complex-2 { yang-version 1.1; - namespace "http://github.com/Azure/static-complex-2"; - prefix static-complex-2; + namespace "http://github.com/Azure/static-complex-2"; + prefix static-complex-2; - container sonic-static-object-complex-2 { - /* sonic-static-object-complex-2 - top level container */ + container sonic-static-object-complex-2 { + /* sonic-static-object-complex-2 - top level container */ - container TABLE_1 { - /* TABLE_1 - table container */ + container TABLE_1 { + /* TABLE_1 - table container */ - description "TABLE_1 description"; + description "TABLE_1 description"; - container OBJECT_1 { - /* OBJECT_1 - object container, it have: - * 2 leafs, - * 2 leaf-lists, - * 2 choices - */ + container OBJECT_1 { + /* OBJECT_1 - object container, it have: + * 2 leafs, + * 2 leaf-lists, + * 2 choices + */ - description "OBJECT_1 description"; + description "OBJECT_1 description"; - leaf OBJ_1_LEAF_1 { - description "OBJ_1_LEAF_1 description"; - type string; - } - - leaf OBJ_1_LEAF_2 { - description "OBJ_1_LEAF_2 description"; - type string; - } + leaf OBJ_1_LEAF_1 { + description "OBJ_1_LEAF_1 description"; + type string; + } + + leaf OBJ_1_LEAF_2 { + description "OBJ_1_LEAF_2 description"; + type string; + } - leaf-list OBJ_1_LEAF_LIST_1 { - type string; - } + leaf-list OBJ_1_LEAF_LIST_1 { + type string; + } - leaf-list OBJ_1_LEAF_LIST_2 { - type string; - } + leaf-list OBJ_1_LEAF_LIST_2 { + type string; + } - choice OBJ_1_CHOICE_1 { - case OBJ_1_CHOICE_1_CASE_1 { - leaf OBJ_1_CHOICE_1_LEAF_1 { - type uint16; - } - } - case OBJ_1_CHOICE_1_CASE_2 { - leaf OBJ_1_CHOICE_1_LEAF_2 { - type string; - } - } - } + choice OBJ_1_CHOICE_1 { + case OBJ_1_CHOICE_1_CASE_1 { + leaf OBJ_1_CHOICE_1_LEAF_1 { + type uint16; + } + } + case OBJ_1_CHOICE_1_CASE_2 { + leaf OBJ_1_CHOICE_1_LEAF_2 { + type string; + } + } + } - choice OBJ_1_CHOICE_2 { - case OBJ_1_CHOICE_2_CASE_1 { - leaf OBJ_1_CHOICE_2_LEAF_1 { - type uint16; - } - } - case OBJ_1_CHOICE_2_CASE_2 { - leaf OBJ_1_CHOICE_2_LEAF_2 { - type string; - } - } - } - } - } - } -} \ No newline at end of file + choice OBJ_1_CHOICE_2 { + case OBJ_1_CHOICE_2_CASE_1 { + leaf OBJ_1_CHOICE_2_LEAF_1 { + type uint16; + } + } + case OBJ_1_CHOICE_2_CASE_2 { + leaf OBJ_1_CHOICE_2_LEAF_2 { + type string; + } + } + } + } + } + } +} From 5fec4a7b3004c22fe413c211a5bc83c42ba07cd5 Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Tue, 31 Aug 2021 09:07:45 +0000 Subject: [PATCH 154/173] Removed unnecessary config_db.json file Signed-off-by: Vadym Hlushko --- tests/cli_autogen_input/config_db.json | 544 ------------------------- tests/cli_autogen_yang_parser_test.py | 2 +- 2 files changed, 1 insertion(+), 545 deletions(-) delete mode 100644 tests/cli_autogen_input/config_db.json diff --git a/tests/cli_autogen_input/config_db.json b/tests/cli_autogen_input/config_db.json deleted file mode 100644 index 5473d6158a..0000000000 --- a/tests/cli_autogen_input/config_db.json +++ /dev/null @@ -1,544 +0,0 @@ -{ - "COPP_GROUP": { - "default": { - "cbs": "600", - "cir": "600", - "meter_type": "packets", - "mode": "sr_tcm", - "queue": "0", - "red_action": "drop" - }, - "queue1_group1": { - "cbs": "6000", - "cir": "6000", - "meter_type": "packets", - "mode": "sr_tcm", - "queue": "1", - "red_action": "drop", - "trap_action": "trap", - "trap_priority": "1" - }, - "queue1_group2": { - "cbs": "600", - "cir": "600", - "meter_type": "packets", - "mode": "sr_tcm", - "queue": "1", - "red_action": "drop", - "trap_action": "trap", - "trap_priority": "1" - }, - "queue2_group1": { - "cbs": "1000", - "cir": "1000", - "genetlink_mcgrp_name": "packets", - "genetlink_name": "psample", - "meter_type": "packets", - "mode": "sr_tcm", - "queue": "2", - "red_action": "drop", - "trap_action": "trap", - "trap_priority": "1" - }, - "queue4_group1": { - "cbs": "600", - "cir": "600", - "color": "blind", - "meter_type": "packets", - "mode": "sr_tcm", - "queue": "4", - "red_action": "drop", - "trap_action": "trap", - "trap_priority": "4" - }, - "queue4_group2": { - "cbs": "600", - "cir": "600", - "meter_type": "packets", - "mode": "sr_tcm", - "queue": "4", - "red_action": "drop", - "trap_action": "copy", - "trap_priority": "4" - }, - "queue4_group3": { - "cbs": "600", - "cir": "600", - "color": "blind", - "meter_type": "packets", - "mode": "sr_tcm", - "queue": "4", - "red_action": "drop", - "trap_action": "trap", - "trap_priority": "4" - } - }, - "COPP_TRAP": { - "arp": { - "trap_group": "queue4_group2", - "trap_ids": "arp_req,arp_resp,neigh_discovery" - }, - "bgp": { - "trap_group": "queue4_group1", - "trap_ids": "bgp,bgpv6" - }, - "dhcp": { - "trap_group": "queue4_group3", - "trap_ids": "dhcp,dhcpv6" - }, - "ip2me": { - "trap_group": "queue1_group1", - "trap_ids": "ip2me" - }, - "lacp": { - "trap_group": "queue4_group1", - "trap_ids": "lacp" - }, - "lldp": { - "trap_group": "queue4_group3", - "trap_ids": "lldp" - }, - "nat": { - "trap_group": "queue1_group2", - "trap_ids": "src_nat_miss,dest_nat_miss" - }, - "sflow": { - "trap_group": "queue2_group1", - "trap_ids": "sample_packet" - }, - "ssh": { - "trap_group": "queue4_group2", - "trap_ids": "ssh" - }, - "udld": { - "trap_group": "queue4_group3", - "trap_ids": "udld" - } - }, - "CRM": { - "Config": { - "acl_counter_high_threshold": "85", - "acl_counter_low_threshold": "70", - "acl_counter_threshold_type": "percentage", - "acl_entry_high_threshold": "85", - "acl_entry_low_threshold": "70", - "acl_entry_threshold_type": "percentage", - "acl_group_high_threshold": "85", - "acl_group_low_threshold": "70", - "acl_group_threshold_type": "percentage", - "acl_table_high_threshold": "85", - "acl_table_low_threshold": "70", - "acl_table_threshold_type": "percentage", - "dnat_entry_high_threshold": "85", - "dnat_entry_low_threshold": "70", - "dnat_entry_threshold_type": "percentage", - "fdb_entry_high_threshold": "85", - "fdb_entry_low_threshold": "70", - "fdb_entry_threshold_type": "percentage", - "ipmc_entry_high_threshold": "85", - "ipmc_entry_low_threshold": "70", - "ipmc_entry_threshold_type": "percentage", - "ipv4_neighbor_high_threshold": "85", - "ipv4_neighbor_low_threshold": "70", - "ipv4_neighbor_threshold_type": "percentage", - "ipv4_nexthop_high_threshold": "85", - "ipv4_nexthop_low_threshold": "70", - "ipv4_nexthop_threshold_type": "percentage", - "ipv4_route_high_threshold": "85", - "ipv4_route_low_threshold": "70", - "ipv4_route_threshold_type": "percentage", - "ipv6_neighbor_high_threshold": "85", - "ipv6_neighbor_low_threshold": "70", - "ipv6_neighbor_threshold_type": "percentage", - "ipv6_nexthop_high_threshold": "85", - "ipv6_nexthop_low_threshold": "70", - "ipv6_nexthop_threshold_type": "percentage", - "ipv6_route_high_threshold": "85", - "ipv6_route_low_threshold": "70", - "ipv6_route_threshold_type": "percentage", - "nexthop_group_high_threshold": "85", - "nexthop_group_low_threshold": "70", - "nexthop_group_member_high_threshold": "85", - "nexthop_group_member_low_threshold": "70", - "nexthop_group_member_threshold_type": "percentage", - "nexthop_group_threshold_type": "percentage", - "polling_interval": "300", - "snat_entry_high_threshold": "85", - "snat_entry_low_threshold": "70", - "snat_entry_threshold_type": "percentage" - } - }, - "DEVICE_METADATA": { - "localhost": { - "buffer_model": "traditional", - "default_bgp_status": "up", - "default_pfcwd_status": "disable", - "hostname": "r-bulldog-02", - "hwsku": "ACS-MSN2100", - "mac": "98:03:9b:f8:e7:c0", - "platform": "x86_64-mlnx_msn2100-r0", - "type": "ToRRouter" - } - }, - "FEATURE": { - "bgp": { - "auto_restart": "enabled", - "has_global_scope": "False", - "has_per_asic_scope": "True", - "has_timer": "False", - "high_mem_alert": "disabled", - "state": "enabled" - }, - "database": { - "auto_restart": "disabled", - "has_global_scope": "True", - "has_per_asic_scope": "True", - "has_timer": "False", - "high_mem_alert": "disabled", - "state": "enabled" - }, - "dhcp_relay": { - "auto_restart": "enabled", - "has_global_scope": "True", - "has_per_asic_scope": "False", - "has_timer": "False", - "high_mem_alert": "disabled", - "state": "enabled" - }, - "lldp": { - "auto_restart": "enabled", - "has_global_scope": "True", - "has_per_asic_scope": "True", - "has_timer": "False", - "high_mem_alert": "disabled", - "state": "enabled", - "status": "enabled" - }, - "mgmt-framework": { - "auto_restart": "enabled", - "has_global_scope": "True", - "has_per_asic_scope": "False", - "has_timer": "True", - "high_mem_alert": "disabled", - "state": "enabled" - }, - "nat": { - "auto_restart": "enabled", - "has_global_scope": "True", - "has_per_asic_scope": "False", - "has_timer": "False", - "high_mem_alert": "disabled", - "state": "disabled" - }, - "pmon": { - "auto_restart": "enabled", - "has_global_scope": "True", - "has_per_asic_scope": "False", - "has_timer": "False", - "high_mem_alert": "disabled", - "state": "enabled" - }, - "radv": { - "auto_restart": "enabled", - "has_global_scope": "True", - "has_per_asic_scope": "False", - "has_timer": "False", - "high_mem_alert": "disabled", - "state": "enabled" - }, - "sflow": { - "auto_restart": "enabled", - "has_global_scope": "True", - "has_per_asic_scope": "False", - "has_timer": "False", - "high_mem_alert": "disabled", - "state": "disabled" - }, - "snmp": { - "auto_restart": "enabled", - "has_global_scope": "True", - "has_per_asic_scope": "False", - "has_timer": "True", - "high_mem_alert": "disabled", - "state": "enabled" - }, - "swss": { - "auto_restart": "enabled", - "has_global_scope": "False", - "has_per_asic_scope": "True", - "has_timer": "False", - "high_mem_alert": "disabled", - "state": "enabled" - }, - "syncd": { - "auto_restart": "enabled", - "has_global_scope": "False", - "has_per_asic_scope": "True", - "has_timer": "False", - "high_mem_alert": "disabled", - "state": "enabled" - }, - "teamd": { - "auto_restart": "enabled", - "has_global_scope": "False", - "has_per_asic_scope": "True", - "has_timer": "False", - "high_mem_alert": "disabled", - "state": "enabled" - }, - "telemetry": { - "auto_restart": "enabled", - "has_global_scope": "True", - "has_per_asic_scope": "False", - "has_timer": "True", - "high_mem_alert": "disabled", - "state": "enabled" - }, - "what-just-happened": { - "auto_restart": "disabled", - "has_timer": "True", - "high_mem_alert": "disabled", - "state": "enabled" - } - }, - "FLEX_COUNTER_TABLE": { - "BUFFER_POOL_WATERMARK": { - "FLEX_COUNTER_STATUS": "enable" - }, - "PFCWD": { - "FLEX_COUNTER_STATUS": "enable" - }, - "PG_WATERMARK": { - "FLEX_COUNTER_STATUS": "enable" - }, - "PORT": { - "FLEX_COUNTER_STATUS": "enable" - }, - "PORT_BUFFER_DROP": { - "FLEX_COUNTER_STATUS": "enable" - }, - "QUEUE": { - "FLEX_COUNTER_STATUS": "enable" - }, - "QUEUE_WATERMARK": { - "FLEX_COUNTER_STATUS": "enable" - }, - "RIF": { - "FLEX_COUNTER_STATUS": "enable" - } - }, - "KDUMP": { - "config": { - "enabled": "false", - "num_dumps": "3", - "memory": "0M-2G:256M,2G-4G:320M,4G-8G:384M,8G-:448M" - } - }, - "MGMT_INTERFACE": { - "eth0|10.210.25.44/22": { - "gwaddr": "10.210.24.1" - } - }, - "PORT": { - "Ethernet0": { - "admin_status": "up", - "alias": "etp1", - "index": "1", - "lanes": "0,1,2,3", - "speed": "100000" - }, - "Ethernet12": { - "admin_status": "up", - "alias": "etp4a", - "index": "4", - "lanes": "12,13", - "speed": "50000" - }, - "Ethernet14": { - "admin_status": "up", - "alias": "etp4b", - "index": "4", - "lanes": "14,15", - "speed": "50000" - }, - "Ethernet16": { - "admin_status": "up", - "alias": "etp5a", - "index": "5", - "lanes": "16,17", - "speed": "50000" - }, - "Ethernet18": { - "admin_status": "up", - "alias": "etp5b", - "index": "5", - "lanes": "18,19", - "speed": "50000" - }, - "Ethernet20": { - "admin_status": "up", - "alias": "etp6a", - "index": "6", - "lanes": "20", - "speed": "25000" - }, - "Ethernet21": { - "admin_status": "up", - "alias": "etp6b", - "index": "6", - "lanes": "21", - "speed": "25000" - }, - "Ethernet22": { - "admin_status": "up", - "alias": "etp6c", - "index": "6", - "lanes": "22", - "speed": "25000" - }, - "Ethernet23": { - "admin_status": "up", - "alias": "etp6d", - "index": "6", - "lanes": "23", - "speed": "25000" - }, - "Ethernet24": { - "admin_status": "up", - "alias": "etp7a", - "index": "7", - "lanes": "24", - "speed": "25000" - }, - "Ethernet25": { - "admin_status": "up", - "alias": "etp7b", - "index": "7", - "lanes": "25", - "speed": "25000" - }, - "Ethernet26": { - "admin_status": "up", - "alias": "etp7c", - "index": "7", - "lanes": "26", - "speed": "25000" - }, - "Ethernet27": { - "admin_status": "up", - "alias": "etp7d", - "index": "7", - "lanes": "27", - "speed": "25000" - }, - "Ethernet28": { - "admin_status": "up", - "alias": "etp8", - "index": "8", - "lanes": "28,29,30,31", - "speed": "100000" - }, - "Ethernet32": { - "admin_status": "up", - "alias": "etp9", - "index": "9", - "lanes": "32,33,34,35", - "speed": "100000" - }, - "Ethernet36": { - "admin_status": "up", - "alias": "etp10", - "index": "10", - "lanes": "36,37,38,39", - "speed": "100000" - }, - "Ethernet4": { - "admin_status": "up", - "alias": "etp2", - "index": "2", - "lanes": "4,5,6,7", - "speed": "100000" - }, - "Ethernet40": { - "admin_status": "up", - "alias": "etp11", - "index": "11", - "lanes": "40,41,42,43", - "speed": "100000" - }, - "Ethernet44": { - "admin_status": "up", - "alias": "etp12", - "index": "12", - "lanes": "44,45,46,47", - "speed": "100000" - }, - "Ethernet48": { - "admin_status": "up", - "alias": "etp13", - "index": "13", - "lanes": "48,49,50,51", - "speed": "100000" - }, - "Ethernet52": { - "admin_status": "up", - "alias": "etp14", - "index": "14", - "lanes": "52,53,54,55", - "speed": "100000" - }, - "Ethernet56": { - "admin_status": "up", - "alias": "etp15", - "index": "15", - "lanes": "56,57,58,59", - "speed": "100000" - }, - "Ethernet60": { - "admin_status": "up", - "alias": "etp16", - "index": "16", - "lanes": "60,61,62,63", - "speed": "100000" - }, - "Ethernet8": { - "admin_status": "up", - "alias": "etp3", - "index": "3", - "lanes": "8,9,10,11", - "speed": "100000" - } - }, - "SNMP": { - "LOCATION": { - "Location": "public" - } - }, - "SNMP_COMMUNITY": { - "public": { - "TYPE": "RO" - } - }, - "VERSIONS": { - "DATABASE": { - "VERSION": "version_2_0_0" - } - }, - "WJH": { - "global": { - "mode": "debug", - "nice_level": "1", - "pci_bandwidth": "50" - } - }, - "WJH_CHANNEL": { - "forwarding": { - "drop_category_list": "L2,L3,Tunnel", - "type": "raw_and_aggregated" - }, - "layer-1": { - "drop_category_list": "L1", - "type": "raw_and_aggregated" - } - } -} diff --git a/tests/cli_autogen_yang_parser_test.py b/tests/cli_autogen_yang_parser_test.py index 9ed915c69b..2e0eee6c20 100644 --- a/tests/cli_autogen_yang_parser_test.py +++ b/tests/cli_autogen_yang_parser_test.py @@ -153,7 +153,7 @@ def base_test(yang_model_name, correct_dict): """ General logic for each test case """ config_db_path = os.path.join(test_path, - 'cli_autogen_input/config_db.json') + 'mock_tables/config_db.json') parser = YangParser(yang_model_name=yang_model_name, config_db_path=config_db_path, allow_tbl_without_yang=True, From 3cd8ad9a2b5cb927c3a15dfe9f765237f1d73286 Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Tue, 7 Sep 2021 09:52:44 +0000 Subject: [PATCH 155/173] Added test case, added support for UT for sonic-cli-gen implementation Signed-off-by: Vadym Hlushko --- sonic_cli_gen/generator.py | 36 +++++++++++++++++++------- tests/cli_autogen_input/config_db.json | 14 ++++++++++ tests/cli_autogen_test.py | 35 +++++++++++++++++++++++++ tests/cli_autogen_yang_parser_test.py | 2 +- 4 files changed, 76 insertions(+), 11 deletions(-) create mode 100644 tests/cli_autogen_input/config_db.json create mode 100644 tests/cli_autogen_test.py diff --git a/sonic_cli_gen/generator.py b/sonic_cli_gen/generator.py index 4f48b0201a..c4a6ddf64e 100644 --- a/sonic_cli_gen/generator.py +++ b/sonic_cli_gen/generator.py @@ -6,7 +6,10 @@ from sonic_cli_gen.yang_parser import YangParser -templates_path = '/usr/share/sonic/templates/sonic-cli-gen/' +templates_path_switch = '/usr/share/sonic/templates/sonic-cli-gen/' + +config_db_path_ut = '/sonic/src/sonic-utilities/tests/cli_autogen_input/config_db.json' +templates_path_ut = '/sonic/src/sonic-utilities/sonic-utilities-data/templates/sonic-cli-gen/' class CliGenerator: @@ -15,41 +18,54 @@ class CliGenerator: show CLI plugins. Attributes: - loader: the loaded j2 templates - env: j2 central object logger: logger """ def __init__(self, logger): """ Initialize CliGenerator. """ - self.loader = jinja2.FileSystemLoader(templates_path) - self.env = jinja2.Environment(loader=self.loader) self.logger = logger + def generate_cli_plugin(self, cli_group, plugin_name): """ Generate click CLI plugin and put it to: /usr/local/lib//dist-packages//plugins/auto/ """ - parser = YangParser(yang_model_name=plugin_name, - config_db_path='configDB', - allow_tbl_without_yang=True, - debug=False) + if os.environ["UTILITIES_UNIT_TESTING"] == "2": + config_db_path = config_db_path_ut + loader = jinja2.FileSystemLoader(templates_path_ut) + else: + config_db_path = 'configDB' + loader = jinja2.FileSystemLoader(templates_path_switch) + + parser = YangParser( + yang_model_name=plugin_name, + config_db_path=config_db_path, + allow_tbl_without_yang=True, + debug=False + ) # yang_dict will be used as an input for templates located in # /usr/share/sonic/templates/sonic-cli-gen/ yang_dict = parser.parse_yang_model() + + j2_env = jinja2.Environment(loader=loader) + template = j2_env.get_template(cli_group + '.py.j2') + plugin_path = get_cli_plugin_path(cli_group, plugin_name + '_yang.py') - template = self.env.get_template(cli_group + '.py.j2') + with open(plugin_path, 'w') as plugin_py: plugin_py.write(template.render(yang_dict)) self.logger.info(' Auto-generation successful! Location: {}'.format(plugin_path)) + def remove_cli_plugin(self, cli_group, plugin_name): """ Remove CLI plugin from directory: /usr/local/lib//dist-packages//plugins/auto/ """ + plugin_path = get_cli_plugin_path(cli_group, plugin_name + '_yang.py') + if os.path.exists(plugin_path): os.remove(plugin_path) self.logger.info(' {} was removed.'.format(plugin_path)) diff --git a/tests/cli_autogen_input/config_db.json b/tests/cli_autogen_input/config_db.json new file mode 100644 index 0000000000..d8d2efe021 --- /dev/null +++ b/tests/cli_autogen_input/config_db.json @@ -0,0 +1,14 @@ +{ + "DEVICE_METADATA": { + "localhost": { + "buffer_model": "traditional", + "default_bgp_status": "up", + "default_pfcwd_status": "disable", + "hostname": "r-sonic-01", + "hwsku": "ACS-MSN2100", + "mac": "ff:ff:ff:ff:ff:00", + "platform": "x86_64-mlnx_msn2100-r0", + "type": "ToRRouter" + } + } +} \ No newline at end of file diff --git a/tests/cli_autogen_test.py b/tests/cli_autogen_test.py new file mode 100644 index 0000000000..624de34fc2 --- /dev/null +++ b/tests/cli_autogen_test.py @@ -0,0 +1,35 @@ +import os +import logging +import pprint + +import sonic_cli_gen.main as cli_gen + +from sonic_cli_gen.generator import CliGenerator +from click.testing import CliRunner +from utilities_common.db import Db + +logger = logging.getLogger(__name__) + +test_path = os.path.dirname(os.path.abspath(__file__)) +yang_models_path = '/usr/local/yang-models' + + +class TestCliAutogen: + @classmethod + def setup_class(cls): + logger.info("SETUP") + os.environ['UTILITIES_UNIT_TESTING'] = "2" + + @classmethod + def teardown_class(cls): + logger.info("TEARDOWN") + os.environ['UTILITIES_UNIT_TESTING'] = "0" + + def test_one(self): + runner = CliRunner() + db = Db() + obj = {'config_db': db.cfgdb} + + gen = CliGenerator(logger) + res = gen.generate_cli_plugin('config', 'sonic-device_metadata') + diff --git a/tests/cli_autogen_yang_parser_test.py b/tests/cli_autogen_yang_parser_test.py index 2e0eee6c20..1a49d9a441 100644 --- a/tests/cli_autogen_yang_parser_test.py +++ b/tests/cli_autogen_yang_parser_test.py @@ -31,7 +31,7 @@ class TestYangParser: @classmethod def setup_class(cls): logger.info("SETUP") - os.environ['UTILITIES_UNIT_TESTING'] = "1" + os.environ['UTILITIES_UNIT_TESTING'] = "2" move_yang_models() @classmethod From 607b254c7d631fa65a1a92acbefea06c23463a58 Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Tue, 7 Sep 2021 15:42:03 +0000 Subject: [PATCH 156/173] Added registering of plugins to UT Signed-off-by: Vadym Hlushko --- tests/cli_autogen_test.py | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/tests/cli_autogen_test.py b/tests/cli_autogen_test.py index 624de34fc2..a9662a56d4 100644 --- a/tests/cli_autogen_test.py +++ b/tests/cli_autogen_test.py @@ -2,7 +2,12 @@ import logging import pprint -import sonic_cli_gen.main as cli_gen +import show.plugins as show_plugins + +import show.main as show +import config.main as config + +from utilities_common import util_base from sonic_cli_gen.generator import CliGenerator from click.testing import CliRunner @@ -13,23 +18,35 @@ test_path = os.path.dirname(os.path.abspath(__file__)) yang_models_path = '/usr/local/yang-models' +show_device_metadata_localhost="""\ +HWSKU DEFAULT BGP STATUS DOCKER ROUTING CONFIG MODE HOSTNAME PLATFORM MAC DEFAULT PFCWD STATUS BGP ASN DEPLOYMENT ID TYPE BUFFER MODEL FRR MGMT FRAMEWORK CONFIG +---------------------- -------------------- ---------------------------- ------------ ---------------------- ----------------- ---------------------- --------- --------------- --------- -------------- --------------------------- +Mellanox-SN3800-D112C8 down separated sonic-switch x86_64-mlnx_msn3800-r0 1d:34:db:16:a6:00 enable N/A 1 ToRRouter N/A N/A +""" + +gen = CliGenerator(logger) class TestCliAutogen: @classmethod def setup_class(cls): logger.info("SETUP") os.environ['UTILITIES_UNIT_TESTING'] = "2" + gen.generate_cli_plugin('show', 'sonic-device_metadata') + helper = util_base.UtilHelper() + for plugin in helper.load_plugins(show_plugins): + helper.register_plugin(plugin, show.cli) @classmethod def teardown_class(cls): logger.info("TEARDOWN") + gen.remove_cli_plugin('show', 'sonic-device_metadata') os.environ['UTILITIES_UNIT_TESTING'] = "0" def test_one(self): runner = CliRunner() - db = Db() - obj = {'config_db': db.cfgdb} - gen = CliGenerator(logger) - res = gen.generate_cli_plugin('config', 'sonic-device_metadata') + result = runner.invoke(show.cli.commands['device-metadata'].commands['localhost'], []) + logger.debug(result.output) + assert result.output == show_device_metadata_localhost + From 6816903420e04cd9af3a491e88f699267959838c Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Tue, 7 Sep 2021 15:45:03 +0000 Subject: [PATCH 157/173] Fixed name Signed-off-by: Vadym Hlushko --- tests/cli_autogen_test.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/cli_autogen_test.py b/tests/cli_autogen_test.py index a9662a56d4..7d35ff014d 100644 --- a/tests/cli_autogen_test.py +++ b/tests/cli_autogen_test.py @@ -4,8 +4,8 @@ import show.plugins as show_plugins -import show.main as show -import config.main as config +import show.main as show_main +import config.main as config_main from utilities_common import util_base @@ -34,7 +34,7 @@ def setup_class(cls): gen.generate_cli_plugin('show', 'sonic-device_metadata') helper = util_base.UtilHelper() for plugin in helper.load_plugins(show_plugins): - helper.register_plugin(plugin, show.cli) + helper.register_plugin(plugin, show_main.cli) @classmethod def teardown_class(cls): @@ -45,7 +45,7 @@ def teardown_class(cls): def test_one(self): runner = CliRunner() - result = runner.invoke(show.cli.commands['device-metadata'].commands['localhost'], []) + result = runner.invoke(show_main.cli.commands['device-metadata'].commands['localhost'], []) logger.debug(result.output) assert result.output == show_device_metadata_localhost From 2aee8837223d3f709ca685681d4ce205d57c883f Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Wed, 8 Sep 2021 08:47:40 +0000 Subject: [PATCH 158/173] Added helper.load_and_register_plugins() Signed-off-by: Vadym Hlushko --- clear/main.py | 3 +-- config/main.py | 3 +-- show/main.py | 3 +-- tests/cli_autogen_test.py | 25 ++++++++----------------- utilities_common/util_base.py | 6 ++++++ 5 files changed, 17 insertions(+), 23 deletions(-) diff --git a/clear/main.py b/clear/main.py index 4302ae00aa..bb7e475630 100755 --- a/clear/main.py +++ b/clear/main.py @@ -452,8 +452,7 @@ def translations(): # Load plugins and register them helper = util_base.UtilHelper() -for plugin in helper.load_plugins(plugins): - helper.register_plugin(plugin, cli) +helper.load_and_register_plugins(plugins, cli) if __name__ == '__main__': diff --git a/config/main.py b/config/main.py index e9bab3172d..a728c7bd32 100644 --- a/config/main.py +++ b/config/main.py @@ -4562,8 +4562,7 @@ def delete(ctx): # Load plugins and register them helper = util_base.UtilHelper() -for plugin in helper.load_plugins(plugins): - helper.register_plugin(plugin, config) +helper.load_and_register_plugins(plugins, config) if __name__ == '__main__': diff --git a/show/main.py b/show/main.py index b0b2986a78..9afb0217e9 100755 --- a/show/main.py +++ b/show/main.py @@ -1473,8 +1473,7 @@ def ztp(status, verbose): # Load plugins and register them helper = util_base.UtilHelper() -for plugin in helper.load_plugins(plugins): - helper.register_plugin(plugin, cli) +helper.load_and_register_plugins(plugins, cli) if __name__ == '__main__': diff --git a/tests/cli_autogen_test.py b/tests/cli_autogen_test.py index 7d35ff014d..7265ef4df1 100644 --- a/tests/cli_autogen_test.py +++ b/tests/cli_autogen_test.py @@ -3,29 +3,22 @@ import pprint import show.plugins as show_plugins - import show.main as show_main import config.main as config_main from utilities_common import util_base - from sonic_cli_gen.generator import CliGenerator from click.testing import CliRunner -from utilities_common.db import Db logger = logging.getLogger(__name__) - -test_path = os.path.dirname(os.path.abspath(__file__)) -yang_models_path = '/usr/local/yang-models' - -show_device_metadata_localhost="""\ -HWSKU DEFAULT BGP STATUS DOCKER ROUTING CONFIG MODE HOSTNAME PLATFORM MAC DEFAULT PFCWD STATUS BGP ASN DEPLOYMENT ID TYPE BUFFER MODEL FRR MGMT FRAMEWORK CONFIG ----------------------- -------------------- ---------------------------- ------------ ---------------------- ----------------- ---------------------- --------- --------------- --------- -------------- --------------------------- -Mellanox-SN3800-D112C8 down separated sonic-switch x86_64-mlnx_msn3800-r0 1d:34:db:16:a6:00 enable N/A 1 ToRRouter N/A N/A -""" - gen = CliGenerator(logger) +#show_device_metadata_localhost="""\ +#HWSKU DEFAULT BGP STATUS DOCKER ROUTING CONFIG MODE HOSTNAME PLATFORM MAC DEFAULT PFCWD STATUS BGP ASN DEPLOYMENT ID TYPE BUFFER MODEL FRR MGMT FRAMEWORK CONFIG +#---------------------- -------------------- ---------------------------- ------------ ---------------------- ----------------- ---------------------- --------- --------------- --------- -------------- --------------------------- +#Mellanox-SN3800-D112C8 down separated sonic-switch x86_64-mlnx_msn3800-r0 1d:34:db:16:a6:00 enable N/A 1 ToRRouter N/A N/A +#""" + class TestCliAutogen: @classmethod def setup_class(cls): @@ -33,8 +26,7 @@ def setup_class(cls): os.environ['UTILITIES_UNIT_TESTING'] = "2" gen.generate_cli_plugin('show', 'sonic-device_metadata') helper = util_base.UtilHelper() - for plugin in helper.load_plugins(show_plugins): - helper.register_plugin(plugin, show_main.cli) + helper.load_and_register_plugins(show_plugins, show_main.cli) @classmethod def teardown_class(cls): @@ -47,6 +39,5 @@ def test_one(self): result = runner.invoke(show_main.cli.commands['device-metadata'].commands['localhost'], []) logger.debug(result.output) - assert result.output == show_device_metadata_localhost - + #assert result.output == show_device_metadata_localhost diff --git a/utilities_common/util_base.py b/utilities_common/util_base.py index 9bea158b59..98fc230629 100644 --- a/utilities_common/util_base.py +++ b/utilities_common/util_base.py @@ -83,3 +83,9 @@ def check_pddf_mode(self): return True else: return False + + def load_and_register_plugins(self, plugins, cli): + """ Load plugins and register them """ + + for plugin in self.load_plugins(plugins): + self.register_plugin(plugin, cli) \ No newline at end of file From 9fea6dbd4d8c32f6feca493c841e3c3e2248ea60 Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Thu, 9 Sep 2021 11:23:07 +0000 Subject: [PATCH 159/173] Added test cases for sonic-device_metadata Signed-off-by: Vadym Hlushko --- .../autogen_test/show_cmd_output.py | 16 +++ .../autogen_test/sonic-device_metadata.yang | 123 ++++++++++++++++++ tests/cli_autogen_input/config_db.json | 20 ++- tests/cli_autogen_test.py | 91 +++++++++++-- 4 files changed, 227 insertions(+), 23 deletions(-) create mode 100644 tests/cli_autogen_input/autogen_test/show_cmd_output.py create mode 100644 tests/cli_autogen_input/autogen_test/sonic-device_metadata.yang diff --git a/tests/cli_autogen_input/autogen_test/show_cmd_output.py b/tests/cli_autogen_input/autogen_test/show_cmd_output.py new file mode 100644 index 0000000000..cb0632f9ed --- /dev/null +++ b/tests/cli_autogen_input/autogen_test/show_cmd_output.py @@ -0,0 +1,16 @@ +""" +Module holding correct output for the show command for cli_autogen_test.py +""" + + +show_device_metadata_localhost="""\ +HWSKU DEFAULT BGP STATUS DOCKER ROUTING CONFIG MODE HOSTNAME PLATFORM MAC DEFAULT PFCWD STATUS BGP ASN DEPLOYMENT ID TYPE BUFFER MODEL FRR MGMT FRAMEWORK CONFIG +----------- -------------------- ---------------------------- ---------- ---------------------- ----------------- ---------------------- --------- --------------- --------- -------------- --------------------------- +ACS-MSN2100 up N/A r-sonic-01 x86_64-mlnx_msn2100-r0 ff:ff:ff:ff:ff:00 disable N/A N/A ToRRouter traditional N/A +""" + +show_device_metadata_localhost_changed_buffer_model="""\ +HWSKU DEFAULT BGP STATUS DOCKER ROUTING CONFIG MODE HOSTNAME PLATFORM MAC DEFAULT PFCWD STATUS BGP ASN DEPLOYMENT ID TYPE BUFFER MODEL FRR MGMT FRAMEWORK CONFIG +----------- -------------------- ---------------------------- ---------- ---------------------- ----------------- ---------------------- --------- --------------- --------- -------------- --------------------------- +ACS-MSN2100 up N/A r-sonic-01 x86_64-mlnx_msn2100-r0 ff:ff:ff:ff:ff:00 disable N/A N/A ToRRouter dynamic N/A +""" \ No newline at end of file diff --git a/tests/cli_autogen_input/autogen_test/sonic-device_metadata.yang b/tests/cli_autogen_input/autogen_test/sonic-device_metadata.yang new file mode 100644 index 0000000000..400cbf3bcd --- /dev/null +++ b/tests/cli_autogen_input/autogen_test/sonic-device_metadata.yang @@ -0,0 +1,123 @@ +module sonic-device_metadata { + + yang-version 1.1; + + namespace "http://github.com/Azure/sonic-device_metadata"; + prefix device_metadata; + + import ietf-yang-types { + prefix yang; + } + + import ietf-inet-types { + prefix inet; + } + + import sonic-types { + prefix stypes; + revision-date 2019-07-01; + } + + description "DEVICE_METADATA YANG Module for SONiC OS"; + + revision 2021-02-27 { + description "Added frr_mgmt_framework_config field to handle BGP + config DB schema events to configure FRR protocols."; + } + + revision 2020-04-10 { + description "First Revision"; + } + + container sonic-device_metadata { + + container DEVICE_METADATA { + + description "DEVICE_METADATA part of config_db.json"; + + container localhost{ + + leaf hwsku { + type stypes:hwsku; + } + + leaf default_bgp_status { + type enumeration { + enum up; + enum down; + } + default up; + } + + leaf docker_routing_config_mode { + type string { + pattern "unified|split|separated"; + } + default "unified"; + } + + leaf hostname { + type string { + length 1..255; + } + } + + leaf platform { + type string { + length 1..255; + } + } + + leaf mac { + type yang:mac-address; + } + + leaf default_pfcwd_status { + type enumeration { + enum disable; + enum enable; + } + default disable; + } + + leaf bgp_asn { + type inet:as-number; + } + + leaf deployment_id { + type uint32; + } + + leaf type { + type string { + length 1..255; + pattern "ToRRouter|LeafRouter|SpineChassisFrontendRouter|ChassisBackendRouter|ASIC"; + } + } + + leaf buffer_model { + description "This leaf is added for dynamic buffer calculation. + The dynamic model represents the model in which the buffer configurations, + like the headroom sizes and buffer pool sizes, are dynamically calculated based + on the ports' speed, cable length, and MTU. This model is used by Mellanox so far. + The traditional model represents the model in which all the buffer configurations + are statically configured in CONFIG_DB tables. This is the default model used by all other vendors"; + type string { + pattern "dynamic|traditional"; + } + } + + leaf frr_mgmt_framework_config { + type boolean; + description "FRR configurations are handled by sonic-frr-mgmt-framework module when set to true, + otherwise, sonic-bgpcfgd handles the FRR configurations based on the predefined templates."; + default "false"; + } + } + /* end of container localhost */ + } + /* end of container DEVICE_METADATA */ + } + /* end of top level container */ +} +/* end of module sonic-device_metadata */ diff --git a/tests/cli_autogen_input/config_db.json b/tests/cli_autogen_input/config_db.json index d8d2efe021..7b2f72b815 100644 --- a/tests/cli_autogen_input/config_db.json +++ b/tests/cli_autogen_input/config_db.json @@ -1,14 +1,12 @@ { - "DEVICE_METADATA": { - "localhost": { - "buffer_model": "traditional", - "default_bgp_status": "up", - "default_pfcwd_status": "disable", - "hostname": "r-sonic-01", - "hwsku": "ACS-MSN2100", - "mac": "ff:ff:ff:ff:ff:00", - "platform": "x86_64-mlnx_msn2100-r0", - "type": "ToRRouter" - } + "DEVICE_METADATA|localhost": { + "buffer_model": "traditional", + "default_bgp_status": "up", + "default_pfcwd_status": "disable", + "hostname": "r-sonic-01", + "hwsku": "ACS-MSN2100", + "mac": "ff:ff:ff:ff:ff:00", + "platform": "x86_64-mlnx_msn2100-r0", + "type": "ToRRouter" } } \ No newline at end of file diff --git a/tests/cli_autogen_test.py b/tests/cli_autogen_test.py index 7265ef4df1..5d9958454e 100644 --- a/tests/cli_autogen_test.py +++ b/tests/cli_autogen_test.py @@ -1,43 +1,110 @@ import os import logging -import pprint +import pytest import show.plugins as show_plugins import show.main as show_main +import config.plugins as config_plugins import config.main as config_main +from .cli_autogen_input.autogen_test import show_cmd_output from utilities_common import util_base from sonic_cli_gen.generator import CliGenerator +from .mock_tables import dbconnector +from utilities_common.db import Db from click.testing import CliRunner logger = logging.getLogger(__name__) gen = CliGenerator(logger) -#show_device_metadata_localhost="""\ -#HWSKU DEFAULT BGP STATUS DOCKER ROUTING CONFIG MODE HOSTNAME PLATFORM MAC DEFAULT PFCWD STATUS BGP ASN DEPLOYMENT ID TYPE BUFFER MODEL FRR MGMT FRAMEWORK CONFIG -#---------------------- -------------------- ---------------------------- ------------ ---------------------- ----------------- ---------------------- --------- --------------- --------- -------------- --------------------------- -#Mellanox-SN3800-D112C8 down separated sonic-switch x86_64-mlnx_msn3800-r0 1d:34:db:16:a6:00 enable N/A 1 ToRRouter N/A N/A -#""" +test_path = os.path.dirname(os.path.abspath(__file__)) +mock_db_path = os.path.join(test_path, 'cli_autogen_input') + +SUCCESS = 0 +ERROR = 1 +INVALID_VALUE = 'INVALID' + class TestCliAutogen: @classmethod def setup_class(cls): - logger.info("SETUP") + logger.info('SETUP') os.environ['UTILITIES_UNIT_TESTING'] = "2" + gen.generate_cli_plugin('show', 'sonic-device_metadata') + gen.generate_cli_plugin('config', 'sonic-device_metadata') + helper = util_base.UtilHelper() helper.load_and_register_plugins(show_plugins, show_main.cli) + helper.load_and_register_plugins(config_plugins, config_main.config) + @classmethod def teardown_class(cls): - logger.info("TEARDOWN") + logger.info('TEARDOWN') + gen.remove_cli_plugin('show', 'sonic-device_metadata') + gen.remove_cli_plugin('config', 'sonic-device_metadata') + + dbconnector.dedicated_dbs['CONFIG_DB'] = None + os.environ['UTILITIES_UNIT_TESTING'] = "0" - def test_one(self): + + def test_show_device_metadata(self): + dbconnector.dedicated_dbs['CONFIG_DB'] = os.path.join(mock_db_path, 'config_db') + db = Db() runner = CliRunner() - result = runner.invoke(show_main.cli.commands['device-metadata'].commands['localhost'], []) - logger.debug(result.output) - #assert result.output == show_device_metadata_localhost + result = runner.invoke( + show_main.cli.commands['device-metadata'].commands['localhost'], [], obj=db + ) + + logger.debug("\n" + result.output) + logger.debug(result.exit_code) + assert result.exit_code == SUCCESS + assert result.output == show_cmd_output.show_device_metadata_localhost + + + def test_config_device_metadata(self): + dbconnector.dedicated_dbs['CONFIG_DB'] = os.path.join(mock_db_path, 'config_db') + db = Db() + runner = CliRunner() + + result = runner.invoke( + config_main.config.commands['device-metadata'].commands['localhost'].commands['buffer-model'], ['dynamic'], obj=db + ) + + result = runner.invoke( + show_main.cli.commands['device-metadata'].commands['localhost'], [], obj=db + ) + + logger.debug("\n" + result.output) + logger.debug(result.exit_code) + assert result.exit_code == SUCCESS + assert result.output == show_cmd_output.show_device_metadata_localhost_changed_buffer_model + + + @pytest.mark.parametrize("parameter,value", [ + ('default-bgp-status', INVALID_VALUE), + ('docker-routing-config-mode', INVALID_VALUE), + ('mac', INVALID_VALUE), + ('default-pfcwd-status', INVALID_VALUE), + ('bgp-asn', INVALID_VALUE), + ('type', INVALID_VALUE), + ('buffer-model', INVALID_VALUE), + ('frr-mgmt-framework-config', INVALID_VALUE) + ]) + def test_config_device_metadata_invalid(self, parameter, value): + dbconnector.dedicated_dbs['CONFIG_DB'] = os.path.join(mock_db_path, 'config_db') + db = Db() + runner = CliRunner() + + result = runner.invoke( + config_main.config.commands['device-metadata'].commands['localhost'].commands[parameter], [value], obj=db + ) + + logger.debug("\n" + result.output) + logger.debug(result.exit_code) + assert result.exit_code == ERROR From 69926ed309b0254437eed5df3c4be96b7dfce3bd Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Fri, 10 Sep 2021 14:02:48 +0000 Subject: [PATCH 160/173] Added cli_autogen_common.py, added separate dir for each cli_autogen test Signed-off-by: Vadym Hlushko --- tests/cli_autogen_input/cli_autogen_common.py | 24 +++++++ .../assert_dictionaries.py | 0 .../{ => yang_parser_test}/sonic-1-list.yang | 0 .../sonic-1-object-container.yang | 0 .../sonic-1-table-container.yang | 0 .../{ => yang_parser_test}/sonic-2-lists.yang | 0 .../sonic-2-object-containers.yang | 0 .../sonic-2-table-containers.yang | 0 .../sonic-choice-complex.yang | 0 .../sonic-dynamic-object-complex-1.yang | 0 .../sonic-dynamic-object-complex-2.yang | 0 .../sonic-grouping-1.yang | 0 .../sonic-grouping-2.yang | 0 .../sonic-grouping-complex.yang | 0 .../sonic-static-object-complex-1.yang | 0 .../sonic-static-object-complex-2.yang | 0 tests/cli_autogen_yang_parser_test.py | 62 ++++++------------- 17 files changed, 43 insertions(+), 43 deletions(-) create mode 100644 tests/cli_autogen_input/cli_autogen_common.py rename tests/cli_autogen_input/{ => yang_parser_test}/assert_dictionaries.py (100%) rename tests/cli_autogen_input/{ => yang_parser_test}/sonic-1-list.yang (100%) rename tests/cli_autogen_input/{ => yang_parser_test}/sonic-1-object-container.yang (100%) rename tests/cli_autogen_input/{ => yang_parser_test}/sonic-1-table-container.yang (100%) rename tests/cli_autogen_input/{ => yang_parser_test}/sonic-2-lists.yang (100%) rename tests/cli_autogen_input/{ => yang_parser_test}/sonic-2-object-containers.yang (100%) rename tests/cli_autogen_input/{ => yang_parser_test}/sonic-2-table-containers.yang (100%) rename tests/cli_autogen_input/{ => yang_parser_test}/sonic-choice-complex.yang (100%) rename tests/cli_autogen_input/{ => yang_parser_test}/sonic-dynamic-object-complex-1.yang (100%) rename tests/cli_autogen_input/{ => yang_parser_test}/sonic-dynamic-object-complex-2.yang (100%) rename tests/cli_autogen_input/{ => yang_parser_test}/sonic-grouping-1.yang (100%) rename tests/cli_autogen_input/{ => yang_parser_test}/sonic-grouping-2.yang (100%) rename tests/cli_autogen_input/{ => yang_parser_test}/sonic-grouping-complex.yang (100%) rename tests/cli_autogen_input/{ => yang_parser_test}/sonic-static-object-complex-1.yang (100%) rename tests/cli_autogen_input/{ => yang_parser_test}/sonic-static-object-complex-2.yang (100%) diff --git a/tests/cli_autogen_input/cli_autogen_common.py b/tests/cli_autogen_input/cli_autogen_common.py new file mode 100644 index 0000000000..bdc44bfa4f --- /dev/null +++ b/tests/cli_autogen_input/cli_autogen_common.py @@ -0,0 +1,24 @@ +import os + +yang_models_path = '/usr/local/yang-models' + + +def move_yang_models(test_path, test_name, test_yang_models): + """ Move a test YANG models to known location """ + + for yang_model in test_yang_models: + src_path = os.path.join(test_path, + 'cli_autogen_input', + test_name, + yang_model) + cmd = 'sudo cp {} {}'.format(src_path, yang_models_path) + os.system(cmd) + + +def remove_yang_models(test_yang_models): + """ Remove a test YANG models to known location """ + + for yang_model in test_yang_models: + yang_model_path = os.path.join(yang_models_path, yang_model) + cmd = 'sudo rm {}'.format(yang_model_path) + os.system(cmd) diff --git a/tests/cli_autogen_input/assert_dictionaries.py b/tests/cli_autogen_input/yang_parser_test/assert_dictionaries.py similarity index 100% rename from tests/cli_autogen_input/assert_dictionaries.py rename to tests/cli_autogen_input/yang_parser_test/assert_dictionaries.py diff --git a/tests/cli_autogen_input/sonic-1-list.yang b/tests/cli_autogen_input/yang_parser_test/sonic-1-list.yang similarity index 100% rename from tests/cli_autogen_input/sonic-1-list.yang rename to tests/cli_autogen_input/yang_parser_test/sonic-1-list.yang diff --git a/tests/cli_autogen_input/sonic-1-object-container.yang b/tests/cli_autogen_input/yang_parser_test/sonic-1-object-container.yang similarity index 100% rename from tests/cli_autogen_input/sonic-1-object-container.yang rename to tests/cli_autogen_input/yang_parser_test/sonic-1-object-container.yang diff --git a/tests/cli_autogen_input/sonic-1-table-container.yang b/tests/cli_autogen_input/yang_parser_test/sonic-1-table-container.yang similarity index 100% rename from tests/cli_autogen_input/sonic-1-table-container.yang rename to tests/cli_autogen_input/yang_parser_test/sonic-1-table-container.yang diff --git a/tests/cli_autogen_input/sonic-2-lists.yang b/tests/cli_autogen_input/yang_parser_test/sonic-2-lists.yang similarity index 100% rename from tests/cli_autogen_input/sonic-2-lists.yang rename to tests/cli_autogen_input/yang_parser_test/sonic-2-lists.yang diff --git a/tests/cli_autogen_input/sonic-2-object-containers.yang b/tests/cli_autogen_input/yang_parser_test/sonic-2-object-containers.yang similarity index 100% rename from tests/cli_autogen_input/sonic-2-object-containers.yang rename to tests/cli_autogen_input/yang_parser_test/sonic-2-object-containers.yang diff --git a/tests/cli_autogen_input/sonic-2-table-containers.yang b/tests/cli_autogen_input/yang_parser_test/sonic-2-table-containers.yang similarity index 100% rename from tests/cli_autogen_input/sonic-2-table-containers.yang rename to tests/cli_autogen_input/yang_parser_test/sonic-2-table-containers.yang diff --git a/tests/cli_autogen_input/sonic-choice-complex.yang b/tests/cli_autogen_input/yang_parser_test/sonic-choice-complex.yang similarity index 100% rename from tests/cli_autogen_input/sonic-choice-complex.yang rename to tests/cli_autogen_input/yang_parser_test/sonic-choice-complex.yang diff --git a/tests/cli_autogen_input/sonic-dynamic-object-complex-1.yang b/tests/cli_autogen_input/yang_parser_test/sonic-dynamic-object-complex-1.yang similarity index 100% rename from tests/cli_autogen_input/sonic-dynamic-object-complex-1.yang rename to tests/cli_autogen_input/yang_parser_test/sonic-dynamic-object-complex-1.yang diff --git a/tests/cli_autogen_input/sonic-dynamic-object-complex-2.yang b/tests/cli_autogen_input/yang_parser_test/sonic-dynamic-object-complex-2.yang similarity index 100% rename from tests/cli_autogen_input/sonic-dynamic-object-complex-2.yang rename to tests/cli_autogen_input/yang_parser_test/sonic-dynamic-object-complex-2.yang diff --git a/tests/cli_autogen_input/sonic-grouping-1.yang b/tests/cli_autogen_input/yang_parser_test/sonic-grouping-1.yang similarity index 100% rename from tests/cli_autogen_input/sonic-grouping-1.yang rename to tests/cli_autogen_input/yang_parser_test/sonic-grouping-1.yang diff --git a/tests/cli_autogen_input/sonic-grouping-2.yang b/tests/cli_autogen_input/yang_parser_test/sonic-grouping-2.yang similarity index 100% rename from tests/cli_autogen_input/sonic-grouping-2.yang rename to tests/cli_autogen_input/yang_parser_test/sonic-grouping-2.yang diff --git a/tests/cli_autogen_input/sonic-grouping-complex.yang b/tests/cli_autogen_input/yang_parser_test/sonic-grouping-complex.yang similarity index 100% rename from tests/cli_autogen_input/sonic-grouping-complex.yang rename to tests/cli_autogen_input/yang_parser_test/sonic-grouping-complex.yang diff --git a/tests/cli_autogen_input/sonic-static-object-complex-1.yang b/tests/cli_autogen_input/yang_parser_test/sonic-static-object-complex-1.yang similarity index 100% rename from tests/cli_autogen_input/sonic-static-object-complex-1.yang rename to tests/cli_autogen_input/yang_parser_test/sonic-static-object-complex-1.yang diff --git a/tests/cli_autogen_input/sonic-static-object-complex-2.yang b/tests/cli_autogen_input/yang_parser_test/sonic-static-object-complex-2.yang similarity index 100% rename from tests/cli_autogen_input/sonic-static-object-complex-2.yang rename to tests/cli_autogen_input/yang_parser_test/sonic-static-object-complex-2.yang diff --git a/tests/cli_autogen_yang_parser_test.py b/tests/cli_autogen_yang_parser_test.py index 1a49d9a441..ed82693e91 100644 --- a/tests/cli_autogen_yang_parser_test.py +++ b/tests/cli_autogen_yang_parser_test.py @@ -3,27 +3,28 @@ import pprint from sonic_cli_gen.yang_parser import YangParser -from .cli_autogen_input import assert_dictionaries +from .cli_autogen_input.yang_parser_test import assert_dictionaries +from .cli_autogen_input.cli_autogen_common import move_yang_models, remove_yang_models logger = logging.getLogger(__name__) test_path = os.path.dirname(os.path.abspath(__file__)) -yang_models_path = '/usr/local/yang-models' + test_yang_models = [ - 'sonic-1-table-container', - 'sonic-2-table-containers', - 'sonic-1-object-container', - 'sonic-2-object-containers', - 'sonic-1-list', - 'sonic-2-lists', - 'sonic-static-object-complex-1', - 'sonic-static-object-complex-2', - 'sonic-dynamic-object-complex-1', - 'sonic-dynamic-object-complex-2', - 'sonic-choice-complex', - 'sonic-grouping-complex', - 'sonic-grouping-1', - 'sonic-grouping-2', + 'sonic-1-table-container.yang', + 'sonic-2-table-containers.yang', + 'sonic-1-object-container.yang', + 'sonic-2-object-containers.yang', + 'sonic-1-list.yang', + 'sonic-2-lists.yang', + 'sonic-static-object-complex-1.yang', + 'sonic-static-object-complex-2.yang', + 'sonic-dynamic-object-complex-1.yang', + 'sonic-dynamic-object-complex-2.yang', + 'sonic-choice-complex.yang', + 'sonic-grouping-complex.yang', + 'sonic-grouping-1.yang', + 'sonic-grouping-2.yang', ] @@ -32,13 +33,13 @@ class TestYangParser: def setup_class(cls): logger.info("SETUP") os.environ['UTILITIES_UNIT_TESTING'] = "2" - move_yang_models() + move_yang_models(test_path, 'yang_parser_test', test_yang_models) @classmethod def teardown_class(cls): logger.info("TEARDOWN") os.environ['UTILITIES_UNIT_TESTING'] = "0" - remove_yang_models() + remove_yang_models(test_yang_models) def test_1_table_container(self): """ Test for 1 'table' container @@ -163,31 +164,6 @@ def base_test(yang_model_name, correct_dict): assert yang_dict == correct_dict -def move_yang_models(): - """ Move a test YANG models to known location - in order to be parsed by YangParser class - """ - - for yang_model in test_yang_models: - src_path = os.path.join(test_path, - 'cli_autogen_input', - yang_model + '.yang') - cmd = 'sudo cp {} {}'.format(src_path, yang_models_path) - os.system(cmd) - - -def remove_yang_models(): - """ Remove a test YANG models to known location - in order to be parsed by YangParser class - """ - - for yang_model in test_yang_models: - yang_model_path = os.path.join(yang_models_path, - yang_model + '.yang') - cmd = 'sudo rm {}'.format(yang_model_path) - os.system(cmd) - - def pretty_log_debug(dictionary): """ Pretty print of parsed dictionary """ From dd7ec5ea20630a24b8afdd5684529bf5bf1e7745 Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Fri, 10 Sep 2021 14:40:53 +0000 Subject: [PATCH 161/173] Added backup and restore func for yang models Signed-off-by: Vadym Hlushko --- tests/cli_autogen_input/cli_autogen_common.py | 15 +++++++++++++++ tests/cli_autogen_test.py | 4 ++++ 2 files changed, 19 insertions(+) diff --git a/tests/cli_autogen_input/cli_autogen_common.py b/tests/cli_autogen_input/cli_autogen_common.py index bdc44bfa4f..31895d62ad 100644 --- a/tests/cli_autogen_input/cli_autogen_common.py +++ b/tests/cli_autogen_input/cli_autogen_common.py @@ -22,3 +22,18 @@ def remove_yang_models(test_yang_models): yang_model_path = os.path.join(yang_models_path, yang_model) cmd = 'sudo rm {}'.format(yang_model_path) os.system(cmd) + + +def backup_yang_models(): + """ Make a copy of existing YANG models """ + + cmd = 'sudo cp -R {} {}'.format(yang_models_path, yang_models_path + '_backup') + os.system(cmd) + + +def restore_backup_yang_models(): + """ Restore existing YANG models from backup """ + + cmd = 'sudo mv {} {}'.format(yang_models_path + '_backup/', '/usr/local/yang-models') + os.system(cmd) + \ No newline at end of file diff --git a/tests/cli_autogen_test.py b/tests/cli_autogen_test.py index 5d9958454e..cd5933249f 100644 --- a/tests/cli_autogen_test.py +++ b/tests/cli_autogen_test.py @@ -7,6 +7,7 @@ import config.plugins as config_plugins import config.main as config_main from .cli_autogen_input.autogen_test import show_cmd_output +from .cli_autogen_input.cli_autogen_common import backup_yang_models, restore_backup_yang_models from utilities_common import util_base from sonic_cli_gen.generator import CliGenerator @@ -108,3 +109,6 @@ def test_config_device_metadata_invalid(self, parameter, value): logger.debug(result.exit_code) assert result.exit_code == ERROR + def test_one(self): + backup_yang_models() + restore_backup_yang_models() From eb852009d876525d5e91c7f1cfe5fba7538e69fa Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Fri, 10 Sep 2021 14:51:34 +0000 Subject: [PATCH 162/173] Removed if-else UT login from generator.py Signed-off-by: Vadym Hlushko --- sonic_cli_gen/generator.py | 21 ++++++++++----------- tests/cli_autogen_test.py | 24 +++++++++++++++++++----- 2 files changed, 29 insertions(+), 16 deletions(-) diff --git a/sonic_cli_gen/generator.py b/sonic_cli_gen/generator.py index c4a6ddf64e..8f23daa6ac 100644 --- a/sonic_cli_gen/generator.py +++ b/sonic_cli_gen/generator.py @@ -8,8 +8,7 @@ templates_path_switch = '/usr/share/sonic/templates/sonic-cli-gen/' -config_db_path_ut = '/sonic/src/sonic-utilities/tests/cli_autogen_input/config_db.json' -templates_path_ut = '/sonic/src/sonic-utilities/sonic-utilities-data/templates/sonic-cli-gen/' + class CliGenerator: @@ -27,28 +26,28 @@ def __init__(self, logger): self.logger = logger - def generate_cli_plugin(self, cli_group, plugin_name): + def generate_cli_plugin( + self, + cli_group, + plugin_name, + config_db_path='configDB', + templates_path='/usr/share/sonic/templates/sonic-cli-gen/' + ): """ Generate click CLI plugin and put it to: /usr/local/lib//dist-packages//plugins/auto/ """ - if os.environ["UTILITIES_UNIT_TESTING"] == "2": - config_db_path = config_db_path_ut - loader = jinja2.FileSystemLoader(templates_path_ut) - else: - config_db_path = 'configDB' - loader = jinja2.FileSystemLoader(templates_path_switch) - parser = YangParser( yang_model_name=plugin_name, config_db_path=config_db_path, allow_tbl_without_yang=True, debug=False - ) + ) # yang_dict will be used as an input for templates located in # /usr/share/sonic/templates/sonic-cli-gen/ yang_dict = parser.parse_yang_model() + loader = jinja2.FileSystemLoader(templates_path) j2_env = jinja2.Environment(loader=loader) template = j2_env.get_template(cli_group + '.py.j2') diff --git a/tests/cli_autogen_test.py b/tests/cli_autogen_test.py index cd5933249f..846a289a8e 100644 --- a/tests/cli_autogen_test.py +++ b/tests/cli_autogen_test.py @@ -25,6 +25,9 @@ ERROR = 1 INVALID_VALUE = 'INVALID' +config_db_path = '/sonic/src/sonic-utilities/tests/cli_autogen_input/config_db.json' +templates_path = '/sonic/src/sonic-utilities/sonic-utilities-data/templates/sonic-cli-gen/' + class TestCliAutogen: @classmethod @@ -32,8 +35,20 @@ def setup_class(cls): logger.info('SETUP') os.environ['UTILITIES_UNIT_TESTING'] = "2" - gen.generate_cli_plugin('show', 'sonic-device_metadata') - gen.generate_cli_plugin('config', 'sonic-device_metadata') + backup_yang_models() + + gen.generate_cli_plugin( + cli_group='show', + plugin_name='sonic-device_metadata', + config_db_path=config_db_path, + templates_path=templates_path + ) + gen.generate_cli_plugin( + cli_group='config', + plugin_name='sonic-device_metadata', + config_db_path=config_db_path, + templates_path=templates_path + ) helper = util_base.UtilHelper() helper.load_and_register_plugins(show_plugins, show_main.cli) @@ -47,6 +62,8 @@ def teardown_class(cls): gen.remove_cli_plugin('show', 'sonic-device_metadata') gen.remove_cli_plugin('config', 'sonic-device_metadata') + restore_backup_yang_models() + dbconnector.dedicated_dbs['CONFIG_DB'] = None os.environ['UTILITIES_UNIT_TESTING'] = "0" @@ -109,6 +126,3 @@ def test_config_device_metadata_invalid(self, parameter, value): logger.debug(result.exit_code) assert result.exit_code == ERROR - def test_one(self): - backup_yang_models() - restore_backup_yang_models() From 5ccefa615a85ad55381ad8092d3a31604fe28254 Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Mon, 13 Sep 2021 14:50:43 +0000 Subject: [PATCH 163/173] Added couple UT for device-neighbor Signed-off-by: Vadym Hlushko --- .../autogen_test/show_cmd_output.py | 22 ++++ .../autogen_test/sonic-device_neighbor.yang | 78 +++++++++++++ tests/cli_autogen_input/cli_autogen_common.py | 3 +- tests/cli_autogen_input/config_db.json | 53 +++++++++ tests/cli_autogen_test.py | 105 +++++++++++++++--- 5 files changed, 242 insertions(+), 19 deletions(-) create mode 100644 tests/cli_autogen_input/autogen_test/sonic-device_neighbor.yang diff --git a/tests/cli_autogen_input/autogen_test/show_cmd_output.py b/tests/cli_autogen_input/autogen_test/show_cmd_output.py index cb0632f9ed..1ed66b9be1 100644 --- a/tests/cli_autogen_input/autogen_test/show_cmd_output.py +++ b/tests/cli_autogen_input/autogen_test/show_cmd_output.py @@ -13,4 +13,26 @@ HWSKU DEFAULT BGP STATUS DOCKER ROUTING CONFIG MODE HOSTNAME PLATFORM MAC DEFAULT PFCWD STATUS BGP ASN DEPLOYMENT ID TYPE BUFFER MODEL FRR MGMT FRAMEWORK CONFIG ----------- -------------------- ---------------------------- ---------- ---------------------- ----------------- ---------------------- --------- --------------- --------- -------------- --------------------------- ACS-MSN2100 up N/A r-sonic-01 x86_64-mlnx_msn2100-r0 ff:ff:ff:ff:ff:00 disable N/A N/A ToRRouter dynamic N/A +""" + +show_device_neighbor="""\ +PEER NAME NAME MGMT ADDR LOCAL PORT PORT TYPE +----------- -------- ----------- ------------ ------ ------ +Ethernet0 Servers 10.217.0.1 Ethernet0 eth0 type +Ethernet4 Servers0 10.217.0.2 Ethernet4 eth1 type +""" + +show_device_neighbor_added="""\ +PEER NAME NAME MGMT ADDR LOCAL PORT PORT TYPE +----------- -------- ----------- ------------ ------ ------ +Ethernet0 Servers 10.217.0.1 Ethernet0 eth0 type +Ethernet4 Servers0 10.217.0.2 Ethernet4 eth1 type +Ethernet8 Servers1 10.217.0.3 Ethernet8 eth2 type +""" + + +show_device_neighbor_deleted="""\ +PEER NAME NAME MGMT ADDR LOCAL PORT PORT TYPE +----------- -------- ----------- ------------ ------ ------ +Ethernet4 Servers0 10.217.0.2 Ethernet4 eth1 type """ \ No newline at end of file diff --git a/tests/cli_autogen_input/autogen_test/sonic-device_neighbor.yang b/tests/cli_autogen_input/autogen_test/sonic-device_neighbor.yang new file mode 100644 index 0000000000..e1c745dd9a --- /dev/null +++ b/tests/cli_autogen_input/autogen_test/sonic-device_neighbor.yang @@ -0,0 +1,78 @@ +module sonic-device_neighbor { + + yang-version 1.1; + + namespace "http://github.com/Azure/sonic-device_neighbor"; + prefix device_neighbor; + + import ietf-inet-types { + prefix inet; + } + + import sonic-extension { + prefix ext; + revision-date 2019-07-01; + } + + import sonic-port { + prefix port; + revision-date 2019-07-01; + } + + description "DEVICE_NEIGHBOR YANG Module for SONiC OS"; + + revision 2020-04-10 { + description "First Revision"; + } + + container sonic-device_neighbor { + + container DEVICE_NEIGHBOR { + + description "DEVICE_NEIGHBOR part of config_db.json"; + + list DEVICE_NEIGHBOR_LIST { + + key "peer_name"; + + leaf peer_name { + type string { + length 1..255; + } + } + + leaf name { + type string { + length 1..255; + } + } + + leaf mgmt_addr { + type inet:ip-address; + } + + leaf local_port { + type leafref { + path /port:sonic-port/port:PORT/port:PORT_LIST/port:name; + } + } + + leaf port { + type string { + length 1..255; + } + } + + leaf type { + type string { + length 1..255; + } + } + } + /* end of list DEVICE_NEIGHBOR_LIST */ + } + /* end of container DEVICE_NEIGHBOR */ + } + /* end of top level container */ +} +/* end of module sonic-device_neighbor */ diff --git a/tests/cli_autogen_input/cli_autogen_common.py b/tests/cli_autogen_input/cli_autogen_common.py index 31895d62ad..a39602ea22 100644 --- a/tests/cli_autogen_input/cli_autogen_common.py +++ b/tests/cli_autogen_input/cli_autogen_common.py @@ -34,6 +34,7 @@ def backup_yang_models(): def restore_backup_yang_models(): """ Restore existing YANG models from backup """ - cmd = 'sudo mv {} {}'.format(yang_models_path + '_backup/', '/usr/local/yang-models') + cmd = 'sudo cp {} {}'.format(yang_models_path + '_backup/*', yang_models_path) os.system(cmd) + os.system('sudo rm -rf {}'.format(yang_models_path + '_backup')) \ No newline at end of file diff --git a/tests/cli_autogen_input/config_db.json b/tests/cli_autogen_input/config_db.json index 7b2f72b815..5d8c863cec 100644 --- a/tests/cli_autogen_input/config_db.json +++ b/tests/cli_autogen_input/config_db.json @@ -8,5 +8,58 @@ "mac": "ff:ff:ff:ff:ff:00", "platform": "x86_64-mlnx_msn2100-r0", "type": "ToRRouter" + }, + "PORT|Ethernet0": { + "alias": "etp1", + "description": "etp1", + "index": "0", + "lanes": "0, 1, 2, 3", + "mtu": "9100", + "pfc_asym": "off", + "speed": "100000" + }, + "PORT|Ethernet4": { + "admin_status": "up", + "alias": "etp2", + "description": "Servers0:eth0", + "index": "1", + "lanes": "4, 5, 6, 7", + "mtu": "9100", + "pfc_asym": "off", + "speed": "100000" + }, + "PORT|Ethernet8": { + "admin_status": "up", + "alias": "etp3", + "description": "Servers0:eth2", + "index": "2", + "lanes": "8, 9, 10, 11", + "mtu": "9100", + "pfc_asym": "off", + "speed": "100000" + }, + "PORT|Ethernet12": { + "admin_status": "up", + "alias": "etp4", + "description": "Servers0:eth4", + "index": "3", + "lanes": "12, 13, 14, 15", + "mtu": "9100", + "pfc_asym": "off", + "speed": "100000" + }, + "DEVICE_NEIGHBOR|Ethernet0": { + "name": "Servers", + "port": "eth0", + "mgmt_addr": "10.217.0.1", + "local_port": "Ethernet0", + "type": "type" + }, + "DEVICE_NEIGHBOR|Ethernet4": { + "name": "Servers0", + "port": "eth1", + "mgmt_addr": "10.217.0.2", + "local_port": "Ethernet4", + "type": "type" } } \ No newline at end of file diff --git a/tests/cli_autogen_test.py b/tests/cli_autogen_test.py index 846a289a8e..3edbfb7159 100644 --- a/tests/cli_autogen_test.py +++ b/tests/cli_autogen_test.py @@ -7,7 +7,7 @@ import config.plugins as config_plugins import config.main as config_main from .cli_autogen_input.autogen_test import show_cmd_output -from .cli_autogen_input.cli_autogen_common import backup_yang_models, restore_backup_yang_models +from .cli_autogen_input.cli_autogen_common import backup_yang_models, restore_backup_yang_models, move_yang_models, remove_yang_models from utilities_common import util_base from sonic_cli_gen.generator import CliGenerator @@ -28,27 +28,34 @@ config_db_path = '/sonic/src/sonic-utilities/tests/cli_autogen_input/config_db.json' templates_path = '/sonic/src/sonic-utilities/sonic-utilities-data/templates/sonic-cli-gen/' +test_yang_models = [ + 'sonic-device_metadata.yang', + 'sonic-device_neighbor.yang', +] + class TestCliAutogen: @classmethod def setup_class(cls): logger.info('SETUP') - os.environ['UTILITIES_UNIT_TESTING'] = "2" + os.environ['UTILITIES_UNIT_TESTING'] = '2' backup_yang_models() - - gen.generate_cli_plugin( - cli_group='show', - plugin_name='sonic-device_metadata', - config_db_path=config_db_path, - templates_path=templates_path - ) - gen.generate_cli_plugin( - cli_group='config', - plugin_name='sonic-device_metadata', - config_db_path=config_db_path, - templates_path=templates_path - ) + move_yang_models(test_path, 'autogen_test', test_yang_models) + + for yang_model in test_yang_models: + gen.generate_cli_plugin( + cli_group='show', + plugin_name=yang_model.split('.')[0], + config_db_path=config_db_path, + templates_path=templates_path + ) + gen.generate_cli_plugin( + cli_group='config', + plugin_name=yang_model.split('.')[0], + config_db_path=config_db_path, + templates_path=templates_path + ) helper = util_base.UtilHelper() helper.load_and_register_plugins(show_plugins, show_main.cli) @@ -59,14 +66,15 @@ def setup_class(cls): def teardown_class(cls): logger.info('TEARDOWN') - gen.remove_cli_plugin('show', 'sonic-device_metadata') - gen.remove_cli_plugin('config', 'sonic-device_metadata') + for yang_model in test_yang_models: + gen.remove_cli_plugin('show', yang_model.split('.')[0]) + gen.remove_cli_plugin('config', yang_model.split('.')[0]) restore_backup_yang_models() dbconnector.dedicated_dbs['CONFIG_DB'] = None - os.environ['UTILITIES_UNIT_TESTING'] = "0" + os.environ['UTILITIES_UNIT_TESTING'] = '0' def test_show_device_metadata(self): @@ -126,3 +134,64 @@ def test_config_device_metadata_invalid(self, parameter, value): logger.debug(result.exit_code) assert result.exit_code == ERROR + + def test_show_device_neighbor(self): + dbconnector.dedicated_dbs['CONFIG_DB'] = os.path.join(mock_db_path, 'config_db') + db = Db() + runner = CliRunner() + + result = runner.invoke( + show_main.cli.commands['device-neighbor'], [], obj=db + ) + + logger.debug("\n" + result.output) + logger.debug(result.exit_code) + assert show_cmd_output.show_device_neighbor + assert result.exit_code == SUCCESS + + + def test_config_device_neighbor_add(self): + dbconnector.dedicated_dbs['CONFIG_DB'] = os.path.join(mock_db_path, 'config_db') + db = Db() + runner = CliRunner() + + result = runner.invoke( + config_main.config.commands['device-neighbor'].commands['add'], + ['Ethernet8', '--name', 'Servers1', '--mgmt-addr', '10.217.0.3', + '--local-port', 'Ethernet8', '--port', 'eth2', '--type', 'type'], + obj=db + ) + logger.debug("\n" + result.output) + logger.debug(result.exit_code) + + result = runner.invoke( + show_main.cli.commands['device-neighbor'], [], obj=db + ) + + logger.debug("\n" + result.output) + logger.debug(result.exit_code) + assert result.exit_code == SUCCESS + assert result.output == show_cmd_output.show_device_neighbor_added + + + def test_config_device_neighbor_delete(self): + dbconnector.dedicated_dbs['CONFIG_DB'] = os.path.join(mock_db_path, 'config_db') + db = Db() + runner = CliRunner() + + result = runner.invoke( + config_main.config.commands['device-neighbor'].commands['delete'], + ['Ethernet0'], obj=db + ) + logger.debug("\n" + result.output) + logger.debug(result.exit_code) + + result = runner.invoke( + show_main.cli.commands['device-neighbor'], [], obj=db + ) + + logger.debug("\n" + result.output) + logger.debug(result.exit_code) + assert result.exit_code == SUCCESS + assert result.output == show_cmd_output.show_device_neighbor_deleted + From 37ac0b7effdde5f89b4c6bda794a16b1576bd37f Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Tue, 14 Sep 2021 06:37:33 +0000 Subject: [PATCH 164/173] Added UT for update flow Signed-off-by: Vadym Hlushko --- .../autogen_test/show_cmd_output.py | 47 ++++++++++++++++++- tests/cli_autogen_test.py | 47 +++++++++++++++++++ 2 files changed, 92 insertions(+), 2 deletions(-) diff --git a/tests/cli_autogen_input/autogen_test/show_cmd_output.py b/tests/cli_autogen_input/autogen_test/show_cmd_output.py index 1ed66b9be1..19c02c7783 100644 --- a/tests/cli_autogen_input/autogen_test/show_cmd_output.py +++ b/tests/cli_autogen_input/autogen_test/show_cmd_output.py @@ -1,5 +1,5 @@ """ -Module holding correct output for the show command for cli_autogen_test.py +This module are holding correct output for the show command for cli_autogen_test.py """ @@ -9,12 +9,14 @@ ACS-MSN2100 up N/A r-sonic-01 x86_64-mlnx_msn2100-r0 ff:ff:ff:ff:ff:00 disable N/A N/A ToRRouter traditional N/A """ + show_device_metadata_localhost_changed_buffer_model="""\ HWSKU DEFAULT BGP STATUS DOCKER ROUTING CONFIG MODE HOSTNAME PLATFORM MAC DEFAULT PFCWD STATUS BGP ASN DEPLOYMENT ID TYPE BUFFER MODEL FRR MGMT FRAMEWORK CONFIG ----------- -------------------- ---------------------------- ---------- ---------------------- ----------------- ---------------------- --------- --------------- --------- -------------- --------------------------- ACS-MSN2100 up N/A r-sonic-01 x86_64-mlnx_msn2100-r0 ff:ff:ff:ff:ff:00 disable N/A N/A ToRRouter dynamic N/A """ + show_device_neighbor="""\ PEER NAME NAME MGMT ADDR LOCAL PORT PORT TYPE ----------- -------- ----------- ------------ ------ ------ @@ -22,6 +24,7 @@ Ethernet4 Servers0 10.217.0.2 Ethernet4 eth1 type """ + show_device_neighbor_added="""\ PEER NAME NAME MGMT ADDR LOCAL PORT PORT TYPE ----------- -------- ----------- ------------ ------ ------ @@ -35,4 +38,44 @@ PEER NAME NAME MGMT ADDR LOCAL PORT PORT TYPE ----------- -------- ----------- ------------ ------ ------ Ethernet4 Servers0 10.217.0.2 Ethernet4 eth1 type -""" \ No newline at end of file +""" + + +show_device_neighbor_updated_mgmt_addr="""\ +PEER NAME NAME MGMT ADDR LOCAL PORT PORT TYPE +----------- -------- ----------- ------------ ------ ------ +Ethernet0 Servers 10.217.0.5 Ethernet0 eth0 type +Ethernet4 Servers0 10.217.0.2 Ethernet4 eth1 type +""" + + +show_device_neighbor_updated_name="""\ +PEER NAME NAME MGMT ADDR LOCAL PORT PORT TYPE +----------- -------- ----------- ------------ ------ ------ +Ethernet0 Servers1 10.217.0.1 Ethernet0 eth0 type +Ethernet4 Servers0 10.217.0.2 Ethernet4 eth1 type +""" + + +show_device_neighbor_updated_local_port="""\ +PEER NAME NAME MGMT ADDR LOCAL PORT PORT TYPE +----------- -------- ----------- ------------ ------ ------ +Ethernet0 Servers 10.217.0.1 Ethernet12 eth0 type +Ethernet4 Servers0 10.217.0.2 Ethernet4 eth1 type +""" + + +show_device_neighbor_updated_port="""\ +PEER NAME NAME MGMT ADDR LOCAL PORT PORT TYPE +----------- -------- ----------- ------------ ------ ------ +Ethernet0 Servers 10.217.0.1 Ethernet0 eth2 type +Ethernet4 Servers0 10.217.0.2 Ethernet4 eth1 type +""" + + +show_device_neighbor_updated_type="""\ +PEER NAME NAME MGMT ADDR LOCAL PORT PORT TYPE +----------- -------- ----------- ------------ ------ ------ +Ethernet0 Servers 10.217.0.1 Ethernet0 eth0 type2 +Ethernet4 Servers0 10.217.0.2 Ethernet4 eth1 type +""" diff --git a/tests/cli_autogen_test.py b/tests/cli_autogen_test.py index 3edbfb7159..442b18ef2c 100644 --- a/tests/cli_autogen_test.py +++ b/tests/cli_autogen_test.py @@ -195,3 +195,50 @@ def test_config_device_neighbor_delete(self): assert result.exit_code == SUCCESS assert result.output == show_cmd_output.show_device_neighbor_deleted + + @pytest.mark.parametrize("parameter,value,output", [ + ('--mgmt-addr', '10.217.0.5', show_cmd_output.show_device_neighbor_updated_mgmt_addr), + ('--name', 'Servers1', show_cmd_output.show_device_neighbor_updated_name), + ('--local-port', 'Ethernet12', show_cmd_output.show_device_neighbor_updated_local_port), + ('--port', 'eth2', show_cmd_output.show_device_neighbor_updated_port), + ('--type', 'type2', show_cmd_output.show_device_neighbor_updated_type), + ]) + def test_config_device_neighbor_update(self, parameter, value, output): + dbconnector.dedicated_dbs['CONFIG_DB'] = os.path.join(mock_db_path, 'config_db') + db = Db() + runner = CliRunner() + + result = runner.invoke( + config_main.config.commands['device-neighbor'].commands['update'], + ['Ethernet0', parameter, value], obj=db + ) + logger.debug("\n" + result.output) + logger.debug(result.exit_code) + + result = runner.invoke( + show_main.cli.commands['device-neighbor'], [], obj=db + ) + + logger.debug("\n" + result.output) + logger.debug(result.exit_code) + assert result.exit_code == SUCCESS + assert result.output == output + + + @pytest.mark.parametrize("parameter,value", [ + ('--mgmt-addr', INVALID_VALUE), + ('--local-port', INVALID_VALUE) + ]) + def test_config_device_neighbor_update_invalid(self, parameter, value): + dbconnector.dedicated_dbs['CONFIG_DB'] = os.path.join(mock_db_path, 'config_db') + db = Db() + runner = CliRunner() + + result = runner.invoke( + config_main.config.commands['device-neighbor'].commands['update'], + ['Ethernet0', parameter, value], obj=db + ) + logger.debug("\n" + result.output) + logger.debug(result.exit_code) + assert result.exit_code == ERROR + From 440220375ac619d4b7bff54507b7fa5c2454ca10 Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Tue, 14 Sep 2021 06:54:33 +0000 Subject: [PATCH 165/173] Fixed codestyle Signed-off-by: Vadym Hlushko --- tests/cli_autogen_input/cli_autogen_common.py | 22 +++++++++---------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/tests/cli_autogen_input/cli_autogen_common.py b/tests/cli_autogen_input/cli_autogen_common.py index a39602ea22..141bceed9c 100644 --- a/tests/cli_autogen_input/cli_autogen_common.py +++ b/tests/cli_autogen_input/cli_autogen_common.py @@ -7,12 +7,13 @@ def move_yang_models(test_path, test_name, test_yang_models): """ Move a test YANG models to known location """ for yang_model in test_yang_models: - src_path = os.path.join(test_path, - 'cli_autogen_input', - test_name, - yang_model) - cmd = 'sudo cp {} {}'.format(src_path, yang_models_path) - os.system(cmd) + src_path = os.path.join( + test_path, + 'cli_autogen_input', + test_name, + yang_model + ) + os.system('sudo cp {} {}'.format(src_path, yang_models_path)) def remove_yang_models(test_yang_models): @@ -20,21 +21,18 @@ def remove_yang_models(test_yang_models): for yang_model in test_yang_models: yang_model_path = os.path.join(yang_models_path, yang_model) - cmd = 'sudo rm {}'.format(yang_model_path) - os.system(cmd) + os.system('sudo rm {}'.format(yang_model_path)) def backup_yang_models(): """ Make a copy of existing YANG models """ - cmd = 'sudo cp -R {} {}'.format(yang_models_path, yang_models_path + '_backup') - os.system(cmd) + os.system('sudo cp -R {} {}'.format(yang_models_path, yang_models_path + '_backup')) def restore_backup_yang_models(): """ Restore existing YANG models from backup """ - cmd = 'sudo cp {} {}'.format(yang_models_path + '_backup/*', yang_models_path) - os.system(cmd) + os.system('sudo cp {} {}'.format(yang_models_path + '_backup/*', yang_models_path)) os.system('sudo rm -rf {}'.format(yang_models_path + '_backup')) \ No newline at end of file From 9d07092d257e10b84f65898bf7e105caa5f68c56 Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Tue, 14 Sep 2021 06:58:43 +0000 Subject: [PATCH 166/173] Fixed codestyle Signed-off-by: Vadym Hlushko --- sonic_cli_gen/generator.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/sonic_cli_gen/generator.py b/sonic_cli_gen/generator.py index 8f23daa6ac..faf3c7d694 100644 --- a/sonic_cli_gen/generator.py +++ b/sonic_cli_gen/generator.py @@ -9,8 +9,6 @@ templates_path_switch = '/usr/share/sonic/templates/sonic-cli-gen/' - - class CliGenerator: """ SONiC CLI generator. This class provides public API for sonic-cli-gen python library. It can generate config, From 5b6a8aab5ab6fd0ce5fe9be74c0e7a88c6dc5e98 Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Tue, 14 Sep 2021 08:12:36 +0000 Subject: [PATCH 167/173] Fixed test path variables Signed-off-by: Vadym Hlushko --- tests/cli_autogen_test.py | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/tests/cli_autogen_test.py b/tests/cli_autogen_test.py index 442b18ef2c..13407d1c13 100644 --- a/tests/cli_autogen_test.py +++ b/tests/cli_autogen_test.py @@ -19,15 +19,14 @@ gen = CliGenerator(logger) test_path = os.path.dirname(os.path.abspath(__file__)) -mock_db_path = os.path.join(test_path, 'cli_autogen_input') +mock_db_path = os.path.join(test_path, 'cli_autogen_input', 'config_db') +config_db_path = os.path.join(test_path, 'cli_autogen_input', 'config_db.json') +templates_path = os.path.join(test_path, '../', 'sonic-utilities-data', 'templates', 'sonic-cli-gen') SUCCESS = 0 ERROR = 1 INVALID_VALUE = 'INVALID' -config_db_path = '/sonic/src/sonic-utilities/tests/cli_autogen_input/config_db.json' -templates_path = '/sonic/src/sonic-utilities/sonic-utilities-data/templates/sonic-cli-gen/' - test_yang_models = [ 'sonic-device_metadata.yang', 'sonic-device_neighbor.yang', @@ -78,7 +77,7 @@ def teardown_class(cls): def test_show_device_metadata(self): - dbconnector.dedicated_dbs['CONFIG_DB'] = os.path.join(mock_db_path, 'config_db') + dbconnector.dedicated_dbs['CONFIG_DB'] = mock_db_path db = Db() runner = CliRunner() @@ -93,7 +92,7 @@ def test_show_device_metadata(self): def test_config_device_metadata(self): - dbconnector.dedicated_dbs['CONFIG_DB'] = os.path.join(mock_db_path, 'config_db') + dbconnector.dedicated_dbs['CONFIG_DB'] = mock_db_path db = Db() runner = CliRunner() @@ -122,7 +121,7 @@ def test_config_device_metadata(self): ('frr-mgmt-framework-config', INVALID_VALUE) ]) def test_config_device_metadata_invalid(self, parameter, value): - dbconnector.dedicated_dbs['CONFIG_DB'] = os.path.join(mock_db_path, 'config_db') + dbconnector.dedicated_dbs['CONFIG_DB'] = mock_db_path db = Db() runner = CliRunner() @@ -136,7 +135,7 @@ def test_config_device_metadata_invalid(self, parameter, value): def test_show_device_neighbor(self): - dbconnector.dedicated_dbs['CONFIG_DB'] = os.path.join(mock_db_path, 'config_db') + dbconnector.dedicated_dbs['CONFIG_DB'] = mock_db_path db = Db() runner = CliRunner() @@ -151,7 +150,7 @@ def test_show_device_neighbor(self): def test_config_device_neighbor_add(self): - dbconnector.dedicated_dbs['CONFIG_DB'] = os.path.join(mock_db_path, 'config_db') + dbconnector.dedicated_dbs['CONFIG_DB'] = mock_db_path db = Db() runner = CliRunner() @@ -175,7 +174,7 @@ def test_config_device_neighbor_add(self): def test_config_device_neighbor_delete(self): - dbconnector.dedicated_dbs['CONFIG_DB'] = os.path.join(mock_db_path, 'config_db') + dbconnector.dedicated_dbs['CONFIG_DB'] = mock_db_path db = Db() runner = CliRunner() @@ -204,7 +203,7 @@ def test_config_device_neighbor_delete(self): ('--type', 'type2', show_cmd_output.show_device_neighbor_updated_type), ]) def test_config_device_neighbor_update(self, parameter, value, output): - dbconnector.dedicated_dbs['CONFIG_DB'] = os.path.join(mock_db_path, 'config_db') + dbconnector.dedicated_dbs['CONFIG_DB'] = mock_db_path db = Db() runner = CliRunner() @@ -230,7 +229,7 @@ def test_config_device_neighbor_update(self, parameter, value, output): ('--local-port', INVALID_VALUE) ]) def test_config_device_neighbor_update_invalid(self, parameter, value): - dbconnector.dedicated_dbs['CONFIG_DB'] = os.path.join(mock_db_path, 'config_db') + dbconnector.dedicated_dbs['CONFIG_DB'] = mock_db_path db = Db() runner = CliRunner() From df1d329d7b8ddddcac5a3965c5bb990ef4cf1cf4 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Thu, 21 Oct 2021 11:30:00 +0000 Subject: [PATCH 168/173] [sonic-cli-gen] add docstrings Signed-off-by: Stepan Blyschak --- .../templates/sonic-cli-gen/config.py.j2 | 96 +++++++++++++++++-- .../templates/sonic-cli-gen/show.py.j2 | 17 +++- 2 files changed, 100 insertions(+), 13 deletions(-) diff --git a/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 b/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 index 402b7e3dd2..53a85c3c41 100644 --- a/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 +++ b/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 @@ -20,14 +20,25 @@ sonic_cfggen = general.load_module_from_source('sonic_cfggen', '/usr/local/bin/s def exit_with_error(*args, **kwargs): - """ Print a message and abort CLI. """ + """ Print a message with click.secho and abort CLI. + + Args: + args: Positional arguments to pass to click.secho + kwargs: Keyword arguments to pass to click.secho + """ click.secho(*args, **kwargs) raise click.Abort() def validate_config_or_raise(cfg): - """ Validate config db data using ConfigMgmt """ + """ Validate config db data using ConfigMgmt. + + Args: + cfg (Dict): Config DB data to validate. + Raises: + Exception: when cfg does not satisfy YANG schema. + """ try: cfg = sonic_cfggen.FormatConverter.to_serialized(cfg) @@ -37,7 +48,16 @@ def validate_config_or_raise(cfg): def add_entry_validated(db, table, key, data): - """ Add new entry in table and validate configuration """ + """ Add new entry in table and validate configuration. + + Args: + db (swsscommon.ConfigDBConnector): Config DB connector obect. + table (str): Table name to add new entry to. + key (Union[str, Tuple]): Key name in the table. + data (Dict): Entry data. + Raises: + Exception: when cfg does not satisfy YANG schema. + """ cfg = db.get_config() cfg.setdefault(table, {}) @@ -53,6 +73,18 @@ def add_entry_validated(db, table, key, data): def update_entry_validated(db, table, key, data, create_if_not_exists=False): """ Update entry in table and validate configuration. If attribute value in data is None, the attribute is deleted. + + Args: + db (swsscommon.ConfigDBConnector): Config DB connector obect. + table (str): Table name to add new entry to. + key (Union[str, Tuple]): Key name in the table. + data (Dict): Entry data. + create_if_not_exists (bool): + In case entry does not exists already a new entry + is not created if this flag is set to False and + creates a new entry if flag is set to True. + Raises: + Exception: when cfg does not satisfy YANG schema. """ cfg = db.get_config() @@ -75,7 +107,15 @@ def update_entry_validated(db, table, key, data, create_if_not_exists=False): def del_entry_validated(db, table, key): - """ Delete entry in table and validate configuration """ + """ Delete entry in table and validate configuration. + + Args: + db (swsscommon.ConfigDBConnector): Config DB connector obect. + table (str): Table name to add new entry to. + key (Union[str, Tuple]): Key name in the table. + Raises: + Exception: when cfg does not satisfy YANG schema. + """ cfg = db.get_config() cfg.setdefault(table, {}) @@ -89,7 +129,17 @@ def del_entry_validated(db, table, key): def add_list_entry_validated(db, table, key, attr, data): - """ Add new entry into list in table and validate configuration""" + """ Add new entry into list in table and validate configuration. + + Args: + db (swsscommon.ConfigDBConnector): Config DB connector obect. + table (str): Table name to add data to. + key (Union[str, Tuple]): Key name in the table. + attr (str): Attribute name which represents a list the data needs to be added to. + data (List): Data list to add to config DB. + Raises: + Exception: when cfg does not satisfy YANG schema. + """ cfg = db.get_config() cfg.setdefault(table, {}) @@ -106,7 +156,17 @@ def add_list_entry_validated(db, table, key, attr, data): def del_list_entry_validated(db, table, key, attr, data): - """ Delete entry from list in table and validate configuration""" + """ Delete entry from list in table and validate configuration. + + Args: + db (swsscommon.ConfigDBConnector): Config DB connector obect. + table (str): Table name to remove data from. + key (Union[str, Tuple]): Key name in the table. + attr (str): Attribute name which represents a list the data needs to be removed from. + data (Dict): Data list to remove from config DB. + Raises: + Exception: when cfg does not satisfy YANG schema. + """ cfg = db.get_config() cfg.setdefault(table, {}) @@ -125,7 +185,16 @@ def del_list_entry_validated(db, table, key, attr, data): def clear_list_entry_validated(db, table, key, attr): - """ Clear list in object and validate configuration""" + """ Clear list in object and validate configuration. + + Args: + db (swsscommon.ConfigDBConnector): Config DB connector obect. + table (str): Table name to remove the list attribute from. + key (Union[str, Tuple]): Key name in the table. + attr (str): Attribute name which represents a list that needs to be removed. + Raises: + Exception: when cfg does not satisfy YANG schema. + """ update_entry_validated(db, table, key, {attr: None}) @@ -235,7 +304,7 @@ E.g: {{ gen_click_arguments(object["keys"] + [attr]) }} @clicommon.pass_db def {{ list_update_group }}_delete( - db, + db, {{ pythonize(object["keys"] + [attr]) }} ): """ Delete {{ attr.name }} in {{ table.name }} """ @@ -262,7 +331,7 @@ E.g: {{ gen_click_arguments(object["keys"]) }} @clicommon.pass_db def {{ list_update_group }}_clear( - db, + db, {{ pythonize(object["keys"]) }} ): """ Clear {{ attr.name }} in {{ table.name }} """ @@ -473,6 +542,15 @@ def {{ table.name }}(): {% endfor %} def register(cli): + """ Register new CLI nodes in root CLI. + + Args: + cli: Root CLI node. + Raises: + Exception: when root CLI already has a command + we are trying to register. + """ + {%- for table in tables %} cli_node = {{ table.name }} if cli_node.name in cli.commands: diff --git a/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 b/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 index 6ee27f2013..2a3d065fdf 100644 --- a/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 +++ b/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 @@ -63,7 +63,7 @@ Jinja2: {"name": "leaf_1"}, {"name": "leaf_2"}, {"name": "leaf_3", "group": "group_0"} - ]) + ]) }} Result: [ @@ -111,15 +111,15 @@ Result: ] {% endmacro %} -{# Generates a list that represents a header in table view. +{# Generates a list that represents a header in table view. E.g: -Jinja2: {{ +Jinja2: {{ gen_header([ {"name": "key"}, {"name": "leaf_1"}, {"name": "leaf_2"}, {"name": "leaf_3", "group": "group_0"} - ]) + ]) }} Result: @@ -237,6 +237,15 @@ def {{ name }}(db): {% endfor %} def register(cli): + """ Register new CLI nodes in root CLI. + + Args: + cli (click.core.Command): Root CLI node. + Raises: + Exception: when root CLI already has a command + we are trying to register. + """ + {%- for table in tables %} cli_node = {{ table.name }} if cli_node.name in cli.commands: From d531699d1802ca5464d3e30104ec6cab13570826 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Wed, 10 Nov 2021 20:17:32 +0200 Subject: [PATCH 169/173] remove sonic_cli_gen PR part Signed-off-by: Stepan Blyschak --- clear/main.py | 4 +- clear/plugins/auto/__init__.py | 0 config/main.py | 3 +- config/plugins/auto/__init__.py | 0 show/main.py | 3 +- show/plugins/auto/__init__.py | 0 .../bash_completion.d/sonic-cli-gen | 8 - sonic-utilities-data/debian/install | 5 +- .../templates/sonic-cli-gen/common.j2 | 3 - .../templates/sonic-cli-gen/config.py.j2 | 559 -------------- .../templates/sonic-cli-gen/show.py.j2 | 254 ------- sonic_cli_gen/__init__.py | 6 - sonic_cli_gen/generator.py | 80 --- sonic_cli_gen/main.py | 51 -- sonic_cli_gen/yang_parser.py | 679 ------------------ .../autogen_test/show_cmd_output.py | 81 --- .../autogen_test/sonic-device_metadata.yang | 123 ---- .../autogen_test/sonic-device_neighbor.yang | 78 -- tests/cli_autogen_input/cli_autogen_common.py | 38 - tests/cli_autogen_input/config_db.json | 65 -- .../yang_parser_test/assert_dictionaries.py | 626 ---------------- .../yang_parser_test/sonic-1-list.yang | 29 - .../sonic-1-object-container.yang | 23 - .../sonic-1-table-container.yang | 17 - .../yang_parser_test/sonic-2-lists.yang | 42 -- .../sonic-2-object-containers.yang | 29 - .../sonic-2-table-containers.yang | 23 - .../sonic-choice-complex.yang | 91 --- .../sonic-dynamic-object-complex-1.yang | 57 -- .../sonic-dynamic-object-complex-2.yang | 84 --- .../yang_parser_test/sonic-grouping-1.yang | 25 - .../yang_parser_test/sonic-grouping-2.yang | 25 - .../sonic-grouping-complex.yang | 96 --- .../sonic-static-object-complex-1.yang | 49 -- .../sonic-static-object-complex-2.yang | 71 -- tests/cli_autogen_test.py | 243 ------- tests/cli_autogen_yang_parser_test.py | 172 ----- utilities_common/util_base.py | 91 --- 38 files changed, 8 insertions(+), 3825 deletions(-) delete mode 100644 clear/plugins/auto/__init__.py delete mode 100644 config/plugins/auto/__init__.py delete mode 100644 show/plugins/auto/__init__.py delete mode 100644 sonic-utilities-data/bash_completion.d/sonic-cli-gen delete mode 100644 sonic-utilities-data/templates/sonic-cli-gen/common.j2 delete mode 100644 sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 delete mode 100644 sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 delete mode 100644 sonic_cli_gen/__init__.py delete mode 100644 sonic_cli_gen/generator.py delete mode 100644 sonic_cli_gen/main.py delete mode 100644 sonic_cli_gen/yang_parser.py delete mode 100644 tests/cli_autogen_input/autogen_test/show_cmd_output.py delete mode 100644 tests/cli_autogen_input/autogen_test/sonic-device_metadata.yang delete mode 100644 tests/cli_autogen_input/autogen_test/sonic-device_neighbor.yang delete mode 100644 tests/cli_autogen_input/cli_autogen_common.py delete mode 100644 tests/cli_autogen_input/config_db.json delete mode 100644 tests/cli_autogen_input/yang_parser_test/assert_dictionaries.py delete mode 100644 tests/cli_autogen_input/yang_parser_test/sonic-1-list.yang delete mode 100644 tests/cli_autogen_input/yang_parser_test/sonic-1-object-container.yang delete mode 100644 tests/cli_autogen_input/yang_parser_test/sonic-1-table-container.yang delete mode 100644 tests/cli_autogen_input/yang_parser_test/sonic-2-lists.yang delete mode 100644 tests/cli_autogen_input/yang_parser_test/sonic-2-object-containers.yang delete mode 100644 tests/cli_autogen_input/yang_parser_test/sonic-2-table-containers.yang delete mode 100644 tests/cli_autogen_input/yang_parser_test/sonic-choice-complex.yang delete mode 100644 tests/cli_autogen_input/yang_parser_test/sonic-dynamic-object-complex-1.yang delete mode 100644 tests/cli_autogen_input/yang_parser_test/sonic-dynamic-object-complex-2.yang delete mode 100644 tests/cli_autogen_input/yang_parser_test/sonic-grouping-1.yang delete mode 100644 tests/cli_autogen_input/yang_parser_test/sonic-grouping-2.yang delete mode 100644 tests/cli_autogen_input/yang_parser_test/sonic-grouping-complex.yang delete mode 100644 tests/cli_autogen_input/yang_parser_test/sonic-static-object-complex-1.yang delete mode 100644 tests/cli_autogen_input/yang_parser_test/sonic-static-object-complex-2.yang delete mode 100644 tests/cli_autogen_test.py delete mode 100644 tests/cli_autogen_yang_parser_test.py delete mode 100644 utilities_common/util_base.py diff --git a/clear/main.py b/clear/main.py index c436f6c1d8..44fa75c726 100755 --- a/clear/main.py +++ b/clear/main.py @@ -484,8 +484,8 @@ def remap_keys(dict): # Load plugins and register them helper = util_base.UtilHelper() -helper.load_and_register_plugins(plugins, cli) - +for plugin in helper.load_plugins(plugins): + helper.register_plugin(plugin, cli) if __name__ == '__main__': cli() diff --git a/clear/plugins/auto/__init__.py b/clear/plugins/auto/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/config/main.py b/config/main.py index 45aa96cd87..e7f03cb4e2 100644 --- a/config/main.py +++ b/config/main.py @@ -5979,7 +5979,8 @@ def smoothing_interval(interval, rates_type): # Load plugins and register them helper = util_base.UtilHelper() -helper.load_and_register_plugins(plugins, config) +for plugin in helper.load_plugins(plugins): + helper.register_plugin(plugin, cli) if __name__ == '__main__': diff --git a/config/plugins/auto/__init__.py b/config/plugins/auto/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/show/main.py b/show/main.py index 1143aea150..f4998218f2 100755 --- a/show/main.py +++ b/show/main.py @@ -1691,7 +1691,8 @@ def ztp(status, verbose): # Load plugins and register them helper = util_base.UtilHelper() -helper.load_and_register_plugins(plugins, cli) +for plugin in helper.load_plugins(plugins): + helper.register_plugin(plugin, cli) if __name__ == '__main__': cli() diff --git a/show/plugins/auto/__init__.py b/show/plugins/auto/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/sonic-utilities-data/bash_completion.d/sonic-cli-gen b/sonic-utilities-data/bash_completion.d/sonic-cli-gen deleted file mode 100644 index 3327f9c513..0000000000 --- a/sonic-utilities-data/bash_completion.d/sonic-cli-gen +++ /dev/null @@ -1,8 +0,0 @@ -_sonic_cli_gen_completion() { - COMPREPLY=( $( env COMP_WORDS="${COMP_WORDS[*]}" \ - COMP_CWORD=$COMP_CWORD \ - _SONIC_CLI_GEN_COMPLETE=complete $1 ) ) - return 0 -} - -complete -F _sonic_cli_gen_completion -o default sonic-cli-gen; diff --git a/sonic-utilities-data/debian/install b/sonic-utilities-data/debian/install index 1f67b78c20..82d087d54d 100644 --- a/sonic-utilities-data/debian/install +++ b/sonic-utilities-data/debian/install @@ -1,3 +1,2 @@ -bash_completion.d/ /etc/ -templates/*.j2 /usr/share/sonic/templates/ -templates/sonic-cli-gen/*.j2 /usr/share/sonic/templates/sonic-cli-gen/ +bash_completion.d/ /etc/ +templates/*.j2 /usr/share/sonic/templates/ diff --git a/sonic-utilities-data/templates/sonic-cli-gen/common.j2 b/sonic-utilities-data/templates/sonic-cli-gen/common.j2 deleted file mode 100644 index 3b83ee5635..0000000000 --- a/sonic-utilities-data/templates/sonic-cli-gen/common.j2 +++ /dev/null @@ -1,3 +0,0 @@ -{% macro cli_name(name) -%} -{{ name|lower|replace("_", "-") }} -{%- endmacro %} diff --git a/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 b/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 deleted file mode 100644 index 53a85c3c41..0000000000 --- a/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 +++ /dev/null @@ -1,559 +0,0 @@ -{%- from "common.j2" import cli_name -%} -""" -Autogenerated config CLI plugin. -{% if source_template is defined %} -Source template: {{ source_template }} -{% endif %} -{% if source_yang_module is defined %} -Source YANG module: {{ source_yang_module }} -{% endif %} -""" - -import click -import utilities_common.cli as clicommon -import utilities_common.general as general -from config import config_mgmt - - -# Load sonic-cfggen from source since /usr/local/bin/sonic-cfggen does not have .py extension. -sonic_cfggen = general.load_module_from_source('sonic_cfggen', '/usr/local/bin/sonic-cfggen') - - -def exit_with_error(*args, **kwargs): - """ Print a message with click.secho and abort CLI. - - Args: - args: Positional arguments to pass to click.secho - kwargs: Keyword arguments to pass to click.secho - """ - - click.secho(*args, **kwargs) - raise click.Abort() - - -def validate_config_or_raise(cfg): - """ Validate config db data using ConfigMgmt. - - Args: - cfg (Dict): Config DB data to validate. - Raises: - Exception: when cfg does not satisfy YANG schema. - """ - - try: - cfg = sonic_cfggen.FormatConverter.to_serialized(cfg) - config_mgmt.ConfigMgmt().loadData(cfg) - except Exception as err: - raise Exception('Failed to validate configuration: {}'.format(err)) - - -def add_entry_validated(db, table, key, data): - """ Add new entry in table and validate configuration. - - Args: - db (swsscommon.ConfigDBConnector): Config DB connector obect. - table (str): Table name to add new entry to. - key (Union[str, Tuple]): Key name in the table. - data (Dict): Entry data. - Raises: - Exception: when cfg does not satisfy YANG schema. - """ - - cfg = db.get_config() - cfg.setdefault(table, {}) - if key in cfg[table]: - raise Exception(f"{key} already exists") - - cfg[table][key] = data - - validate_config_or_raise(cfg) - db.set_entry(table, key, data) - - -def update_entry_validated(db, table, key, data, create_if_not_exists=False): - """ Update entry in table and validate configuration. - If attribute value in data is None, the attribute is deleted. - - Args: - db (swsscommon.ConfigDBConnector): Config DB connector obect. - table (str): Table name to add new entry to. - key (Union[str, Tuple]): Key name in the table. - data (Dict): Entry data. - create_if_not_exists (bool): - In case entry does not exists already a new entry - is not created if this flag is set to False and - creates a new entry if flag is set to True. - Raises: - Exception: when cfg does not satisfy YANG schema. - """ - - cfg = db.get_config() - cfg.setdefault(table, {}) - - if create_if_not_exists: - cfg[table].setdefault(key, {}) - - if key not in cfg[table]: - raise Exception(f"{key} does not exist") - - for attr, value in data.items(): - if value is None and attr in cfg[table][key]: - cfg[table][key].pop(attr) - else: - cfg[table][key][attr] = value - - validate_config_or_raise(cfg) - db.set_entry(table, key, cfg[table][key]) - - -def del_entry_validated(db, table, key): - """ Delete entry in table and validate configuration. - - Args: - db (swsscommon.ConfigDBConnector): Config DB connector obect. - table (str): Table name to add new entry to. - key (Union[str, Tuple]): Key name in the table. - Raises: - Exception: when cfg does not satisfy YANG schema. - """ - - cfg = db.get_config() - cfg.setdefault(table, {}) - if key not in cfg[table]: - raise Exception(f"{key} does not exist") - - cfg[table].pop(key) - - validate_config_or_raise(cfg) - db.set_entry(table, key, None) - - -def add_list_entry_validated(db, table, key, attr, data): - """ Add new entry into list in table and validate configuration. - - Args: - db (swsscommon.ConfigDBConnector): Config DB connector obect. - table (str): Table name to add data to. - key (Union[str, Tuple]): Key name in the table. - attr (str): Attribute name which represents a list the data needs to be added to. - data (List): Data list to add to config DB. - Raises: - Exception: when cfg does not satisfy YANG schema. - """ - - cfg = db.get_config() - cfg.setdefault(table, {}) - if key not in cfg[table]: - raise Exception(f"{key} does not exist") - cfg[table][key].setdefault(attr, []) - for entry in data: - if entry in cfg[table][key][attr]: - raise Exception(f"{entry} already exists") - cfg[table][key][attr].append(entry) - - validate_config_or_raise(cfg) - db.set_entry(table, key, cfg[table][key]) - - -def del_list_entry_validated(db, table, key, attr, data): - """ Delete entry from list in table and validate configuration. - - Args: - db (swsscommon.ConfigDBConnector): Config DB connector obect. - table (str): Table name to remove data from. - key (Union[str, Tuple]): Key name in the table. - attr (str): Attribute name which represents a list the data needs to be removed from. - data (Dict): Data list to remove from config DB. - Raises: - Exception: when cfg does not satisfy YANG schema. - """ - - cfg = db.get_config() - cfg.setdefault(table, {}) - if key not in cfg[table]: - raise Exception(f"{key} does not exist") - cfg[table][key].setdefault(attr, []) - for entry in data: - if entry not in cfg[table][key][attr]: - raise Exception(f"{entry} does not exist") - cfg[table][key][attr].remove(entry) - if not cfg[table][key][attr]: - cfg[table][key].pop(attr) - - validate_config_or_raise(cfg) - db.set_entry(table, key, cfg[table][key]) - - -def clear_list_entry_validated(db, table, key, attr): - """ Clear list in object and validate configuration. - - Args: - db (swsscommon.ConfigDBConnector): Config DB connector obect. - table (str): Table name to remove the list attribute from. - key (Union[str, Tuple]): Key name in the table. - attr (str): Attribute name which represents a list that needs to be removed. - Raises: - Exception: when cfg does not satisfy YANG schema. - """ - - update_entry_validated(db, table, key, {attr: None}) - - -{# Generate click arguments macro -Jinja2 Call: - {{ gen_click_arguments([{"name": "leaf1", "is-leaf-list": False}, - {"name": "leaf2", "is-leaf-list": Talse}) }} -Result: -@click.argument( - "leaf1", - nargs=1, - required=True, -) -@click.argument( - "leaf2", - nargs=-1, - required=True, -) -#} -{%- macro gen_click_arguments(attrs) -%} -{%- for attr in attrs %} -@click.argument( - "{{ cli_name(attr.name) }}", - nargs={% if attr["is-leaf-list"] %}-1{% else %}1{% endif %}, - required=True, -) -{%- endfor %} -{%- endmacro %} - - -{# Generate click options macro -Jinja2 Call: - {{ gen_click_arguments([{"name": "leaf1", "is-mandatory": True, "description": "leaf1-desc"}, - {"name": "leaf2", "is-mandatory": False, "description": "leaf2-desc"}) }} -Result: -@click.option( - "--leaf1", - help="leaf1-desc [mandatory]", -) -@click.option( - "--leaf2", - help="leaf2-desc", -) -#} -{%- macro gen_click_options(attrs) -%} -{%- for attr in attrs %} -@click.option( - "--{{ cli_name(attr.name) }}", - help="{{ attr.description }}{% if attr['is-mandatory'] %}[mandatory]{% endif %}", -) -{%- endfor %} -{%- endmacro %} - -{# Generate valid python identifier from input names #} -{% macro pythonize(attrs) -%} -{{ attrs|map(attribute="name")|map("lower")|map("replace", "-", "_")|join(", ") }} -{%- endmacro %} - -{% macro gen_cfg_obj_list_update(group, table, object, attr) %} -{% set list_update_group = group + "_" + attr.name %} - -@{{ group }}.group(name="{{ cli_name(attr.name) }}", - cls=clicommon.AliasedGroup) -def {{ list_update_group }}(): - """ Add/Delete {{ attr.name }} in {{ table.name }} """ - - pass - -{# Add entries to list attribute config CLI generation -E.g: - @TABLE_object.command(name="add") - @click.argument("key1", nargs=1) - @click.argument("key2", nargs=1) - @click.argument("attribute", nargs=-1) - def TABLE_object_attribute_add(db, key1, key2, attribute): -#} -@{{ list_update_group }}.command(name="add") -{{ gen_click_arguments(object["keys"] + [attr]) }} -@clicommon.pass_db -def {{ list_update_group }}_add( - db, - {{ pythonize(object["keys"] + [attr]) }} -): - """ Add {{ attr.name }} in {{ table.name }} """ - - table = "{{ table.name }}" - key = {{ pythonize(object["keys"]) }} - attr = "{{ attr.name }}" - data = {{ pythonize([attr]) }} - - try: - add_list_entry_validated(db.cfgdb, table, key, attr, data) - except Exception as err: - exit_with_error(f"Error: {err}", fg="red") - - -{# Delete entries from list attribute config CLI generation -E.g: - @TABLE_object.command(name="delete") - @click.argument("key1", nargs=1) - @click.argument("key2", nargs=1) - @click.argument("attribute", nargs=-1) - def TABLE_object_attribute_delete(db, key1, key2, attribute): -#} -@{{ list_update_group }}.command(name="delete") -{{ gen_click_arguments(object["keys"] + [attr]) }} -@clicommon.pass_db -def {{ list_update_group }}_delete( - db, - {{ pythonize(object["keys"] + [attr]) }} -): - """ Delete {{ attr.name }} in {{ table.name }} """ - - table = "{{ table.name }}" - key = {{ pythonize(object["keys"]) }} - attr = "{{ attr.name }}" - data = {{ pythonize([attr]) }} - - try: - del_list_entry_validated(db.cfgdb, table, key, attr, data) - except Exception as err: - exit_with_error(f"Error: {err}", fg="red") - - -{# Clear entries from list attribute config CLI generation -E.g: - @TABLE_object.command(name="delete") - @click.argument("key1", nargs=1) - @click.argument("key2", nargs=1) - def TABLE_object_attribute_clear(db, key1, key2): -#} -@{{ list_update_group }}.command(name="clear") -{{ gen_click_arguments(object["keys"]) }} -@clicommon.pass_db -def {{ list_update_group }}_clear( - db, - {{ pythonize(object["keys"]) }} -): - """ Clear {{ attr.name }} in {{ table.name }} """ - - table = "{{ table.name }}" - key = {{ pythonize(object["keys"]) }} - attr = "{{ attr.name }}" - - try: - clear_list_entry_validated(db.cfgdb, table, key, attr) - except Exception as err: - exit_with_error(f"Error: {err}", fg="red") - -{% endmacro %} - - -{% macro gen_cfg_obj_list_update_all(group, table, object) %} -{% for attr in object.attrs %} -{% if attr["is-leaf-list"] %} -{{ gen_cfg_obj_list_update(group, table, object, attr) }} -{% endif %} -{% endfor %} -{% endmacro %} - - -{% macro gen_cfg_static_obj_attr(table, object, attr) %} -@{{ table.name }}_{{ object.name }}.command(name="{{ cli_name(attr.name) }}") -{{ gen_click_arguments([attr]) }} -@clicommon.pass_db -def {{ table.name }}_{{ object.name }}_{{ attr.name }}(db, {{ pythonize([attr]) }}): - """ {{ attr.description }} """ - - table = "{{ table.name }}" - key = "{{ object.name }}" - data = { - "{{ attr.name }}": {{ pythonize([attr]) }}, - } - try: - update_entry_validated(db.cfgdb, table, key, data, create_if_not_exists=True) - except Exception as err: - exit_with_error(f"Error: {err}", fg="red") -{% endmacro %} - - -{# Static objects config CLI generation -E.g: - @TABLE.group(name="object") - def TABLE_object(db): -#} -{% macro gen_cfg_static_obj(table, object) %} -@{{ table.name }}.group(name="{{ cli_name(object.name) }}", - cls=clicommon.AliasedGroup) -@clicommon.pass_db -def {{ table.name }}_{{ object.name }}(db): - """ {{ object.description }} """ - - pass - -{# Static objects attributes config CLI generation -E.g: - @TABLE_object.command(name="attribute") - def TABLE_object_attribute(db, attribute): -#} -{% for attr in object.attrs %} -{{ gen_cfg_static_obj_attr(table, object, attr) }} -{% endfor %} - -{{ gen_cfg_obj_list_update_all(table.name + "_" + object.name, table, object) }} -{% endmacro %} - -{# Dynamic objects config CLI generation #} - -{# Dynamic objects add command -E.g: - @TABLE.command(name="add") - @click.argument("key1") - @click.argument("key2") - @click.option("--attr1") - @click.option("--attr2") - @click.option("--attr3") - def TABLE_TABLE_LIST_add(db, key1, key2, attr1, attr2, attr3): -#} -{% macro gen_cfg_dyn_obj_add(group, table, object) %} -@{{ group }}.command(name="add") -{{ gen_click_arguments(object["keys"]) }} -{{ gen_click_options(object.attrs) }} -@clicommon.pass_db -def {{ group }}_add(db, {{ pythonize(object["keys"] + object.attrs) }}): - """ Add object in {{ table.name }}. """ - - table = "{{ table.name }}" - key = {{ pythonize(object["keys"]) }} - data = {} -{%- for attr in object.attrs %} - if {{ pythonize([attr]) }} is not None: -{%- if not attr["is-leaf-list"] %} - data["{{ attr.name }}"] = {{ pythonize([attr]) }} -{%- else %} - data["{{ attr.name }}"] = {{ pythonize([attr]) }}.split(",") -{%- endif %} -{%- endfor %} - - try: - add_entry_validated(db.cfgdb, table, key, data) - except Exception as err: - exit_with_error(f"Error: {err}", fg="red") -{% endmacro %} - -{# Dynamic objects update command -E.g: - @TABLE.command(name="update") - @click.argument("key1") - @click.argument("key2") - @click.option("--attr1") - @click.option("--attr2") - @click.option("--attr3") - def TABLE_TABLE_LIST_update(db, key1, key2, attr1, attr2, attr3): -#} -{% macro gen_cfg_dyn_obj_update(group, table, object) %} -@{{ group }}.command(name="update") -{{ gen_click_arguments(object["keys"]) }} -{{ gen_click_options(object.attrs) }} -@clicommon.pass_db -def {{ group }}_update(db, {{ pythonize(object["keys"] + object.attrs) }}): - """ Add object in {{ table.name }}. """ - - table = "{{ table.name }}" - key = {{ pythonize(object["keys"]) }} - data = {} -{%- for attr in object.attrs %} - if {{ pythonize([attr]) }} is not None: -{%- if not attr["is-leaf-list"] %} - data["{{ attr.name }}"] = {{ pythonize([attr]) }} -{%- else %} - data["{{ attr.name }}"] = {{ pythonize([attr]) }}.split(",") -{%- endif %} -{%- endfor %} - - try: - update_entry_validated(db.cfgdb, table, key, data) - except Exception as err: - exit_with_error(f"Error: {err}", fg="red") -{% endmacro %} - -{# Dynamic objects delete command -E.g: - @TABLE.command(name="delete") - @click.argument("key1") - @click.argument("key2") - def TABLE_TABLE_LIST_delete(db, key1, key2): -#} -{% macro gen_cfg_dyn_obj_delete(group, table, object) %} -@{{ group }}.command(name="delete") -{{ gen_click_arguments(object["keys"]) }} -@clicommon.pass_db -def {{ group }}_delete(db, {{ pythonize(object["keys"]) }}): - """ Delete object in {{ table.name }}. """ - - table = "{{ table.name }}" - key = {{ pythonize(object["keys"]) }} - try: - del_entry_validated(db.cfgdb, table, key) - except Exception as err: - exit_with_error(f"Error: {err}", fg="red") -{% endmacro %} - -{% macro gen_cfg_dyn_obj(table, object) %} -{# Generate another nested group in case table holds two types of objects #} -{% if table["dynamic-objects"]|length > 1 %} -{% set group = table.name + "_" + object.name %} -@{{ table.name }}.group(name="{{ cli_name(object.name) }}", - cls=clicommon.AliasedGroup) -def {{ group }}(): - """ {{ object.description }} """ - - pass -{% else %} -{% set group = table.name %} -{% endif %} - -{{ gen_cfg_dyn_obj_add(group, table, object) }} -{{ gen_cfg_dyn_obj_update(group, table, object) }} -{{ gen_cfg_dyn_obj_delete(group, table, object) }} -{{ gen_cfg_obj_list_update_all(group, table, object) }} -{% endmacro %} - - -{% for table in tables %} -@click.group(name="{{ cli_name(table.name) }}", - cls=clicommon.AliasedGroup) -def {{ table.name }}(): - """ {{ table.description }} """ - - pass - -{% if "static-objects" in table %} -{% for object in table["static-objects"] %} -{{ gen_cfg_static_obj(table, object) }} -{% endfor %} -{% endif %} - -{% if "dynamic-objects" in table %} -{% for object in table["dynamic-objects"] %} -{{ gen_cfg_dyn_obj(table, object) }} -{% endfor %} -{% endif %} - -{% endfor %} - -def register(cli): - """ Register new CLI nodes in root CLI. - - Args: - cli: Root CLI node. - Raises: - Exception: when root CLI already has a command - we are trying to register. - """ - -{%- for table in tables %} - cli_node = {{ table.name }} - if cli_node.name in cli.commands: - raise Exception(f"{cli_node.name} already exists in CLI") - cli.add_command({{ table.name }}) -{%- endfor %} diff --git a/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 b/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 deleted file mode 100644 index 2a3d065fdf..0000000000 --- a/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 +++ /dev/null @@ -1,254 +0,0 @@ -{% from "common.j2" import cli_name -%} -""" -Auto-generated show CLI plugin. -{% if source_template is defined %} -Source template: {{ source_template }} -{% endif %} -{% if source_yang_module is defined %} -Source YANG module: {{ source_yang_module }} -{% endif %} -""" - -import click -import tabulate -import natsort -import utilities_common.cli as clicommon - - -{% macro column_name(name) -%} -{{ name|upper|replace("_", " ")|replace("-", " ") }} -{%- endmacro %} - - -def format_attr_value(entry, attr): - """ Helper that formats attribute to be presented in the table output. - - Args: - entry (Dict[str, str]): CONFIG DB entry configuration. - attr (Dict): Attribute metadata. - - Returns: - str: fomatted attribute value. - """ - - if attr["is-leaf-list"]: - return "\n".join(entry.get(attr["name"], [])) - return entry.get(attr["name"], "N/A") - - -def format_group_value(entry, attrs): - """ Helper that formats grouped attribute to be presented in the table output. - - Args: - entry (Dict[str, str]): CONFIG DB entry configuration. - attrs (List[Dict]): Attributes metadata that belongs to the same group. - - Returns: - str: fomatted group attributes. - """ - - data = [] - for attr in attrs: - if entry.get(attr["name"]): - data.append((attr["name"] + ":", format_attr_value(entry, attr))) - return tabulate.tabulate(data, tablefmt="plain") - - -{# Generates a python list that represents a row in the table view. -E.g: -Jinja2: -{{ - gen_row("entry", [ - {"name": "leaf1"}, - {"name": "leaf_1"}, - {"name": "leaf_2"}, - {"name": "leaf_3", "group": "group_0"} - ]) -}} -Result: -[ - format_attr_value( - entry, - {'name': 'leaf1'} - ), - format_attr_value( - entry, - {'name': 'leaf_1'} - ), - format_attr_value( - entry, - {'name': 'leaf_2'} - ), - format_group_value( - entry, - [{'name': 'leaf_3', 'group': 'group_0'}] - ), -] -#} -{% macro gen_row(entry, attrs) -%} -[ -{%- for attr in attrs|rejectattr("group", "defined") %} - format_attr_value( - {{ entry }}, - {{ attr }} - ), -{%- endfor %} -{%- for group, attrs in attrs|selectattr("group", "defined")|groupby("group") %} -{%- if group == "" %} -{%- for attr in attrs %} - format_attr_value( - {{ entry }}, - {{ attr }} - ), -{%- endfor %} -{%- else %} - format_group_value( - {{ entry }}, - {{ attrs }} - ), -{%- endif %} -{%- endfor %} -] -{% endmacro %} - -{# Generates a list that represents a header in table view. -E.g: -Jinja2: {{ - gen_header([ - {"name": "key"}, - {"name": "leaf_1"}, - {"name": "leaf_2"}, - {"name": "leaf_3", "group": "group_0"} - ]) - }} - -Result: -[ - "KEY", - "LEAF 1", - "LEAF 2", - "GROUP 0", -] - -#} -{% macro gen_header(attrs) -%} -[ -{% for attr in attrs|rejectattr("group", "defined") -%} - "{{ column_name(attr.name) }}", -{% endfor -%} -{% for group, attrs in attrs|selectattr("group", "defined")|groupby("group") -%} -{%- if group == "" %} -{% for attr in attrs -%} - "{{ column_name(attr.name) }}", -{% endfor -%} -{%- else %} - "{{ column_name(group) }}", -{%- endif %} -{% endfor -%} -] -{% endmacro %} - - -{% for table in tables %} -{% if "static-objects" in table %} -{# For static objects generate a command group called against table name. -E.g: -@click.group(name="table-name", - cls=clicommon.AliasedGroup) -def TABLE_NAME(): - """ TABLE DESCRIPTION """ - - pass -#} -@click.group(name="{{ cli_name(table.name) }}", - cls=clicommon.AliasedGroup) -def {{ table.name }}(): - """ {{ table.description }} """ - - pass - -{% for object in table["static-objects"] %} -{# For every object in static table generate a command -in the group to show individual object configuration. -CLI command is named against the object key in DB. -E.g: -@TABLE_NAME.command(name="object-name") -@clicommon.pass_db -def TABLE_NAME_object_name(db): - ... -#} -@{{ table.name }}.command(name="{{ cli_name(object.name) }}") -@clicommon.pass_db -def {{ table.name }}_{{ object.name }}(db): - """ {{ object.description }} """ - - header = {{ gen_header(object.attrs) }} - body = [] - - table = db.cfgdb.get_table("{{ table.name }}") - entry = table.get("{{ object.name }}", {}) - row = {{ gen_row("entry", object.attrs) }} - body.append(row) - click.echo(tabulate.tabulate(body, header)) - -{% endfor %} -{% elif "dynamic-objects" in table %} -{% if table["dynamic-objects"]|length > 1 %} -@click.group(name="{{ cli_name(table.name) }}", - cls=clicommon.AliasedGroup) -def {{ table.name }}(): - """ {{ table.description }} """ - - pass -{% endif %} -{% for object in table["dynamic-objects"] %} -{# Generate another nesting group in case table holds two types of objects #} -{% if table["dynamic-objects"]|length > 1 %} -{% set group = table.name %} -{% set name = object.name %} -{% else %} -{% set group = "click" %} -{% set name = table.name %} -{% endif %} - -{# Generate an implementation to display table. #} -@{{ group }}.group(name="{{ cli_name(name) }}", - cls=clicommon.AliasedGroup, - invoke_without_command=True) -@clicommon.pass_db -def {{ name }}(db): - """ {{ object.description }} [Callable command group] """ - - header = {{ gen_header(object["keys"] + object.attrs) }} - body = [] - - table = db.cfgdb.get_table("{{ table.name }}") - for key in natsort.natsorted(table): - entry = table[key] - if not isinstance(key, tuple): - key = (key,) - - row = [*key] + {{ gen_row("entry", object.attrs) }} - body.append(row) - - click.echo(tabulate.tabulate(body, header)) -{% endfor %} -{% endif %} -{% endfor %} - -def register(cli): - """ Register new CLI nodes in root CLI. - - Args: - cli (click.core.Command): Root CLI node. - Raises: - Exception: when root CLI already has a command - we are trying to register. - """ - -{%- for table in tables %} - cli_node = {{ table.name }} - if cli_node.name in cli.commands: - raise Exception(f"{cli_node.name} already exists in CLI") - cli.add_command({{ table.name }}) -{%- endfor %} diff --git a/sonic_cli_gen/__init__.py b/sonic_cli_gen/__init__.py deleted file mode 100644 index e7e775c0fb..0000000000 --- a/sonic_cli_gen/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env python - -from sonic_cli_gen.generator import CliGenerator - -__all__ = ['CliGenerator'] - diff --git a/sonic_cli_gen/generator.py b/sonic_cli_gen/generator.py deleted file mode 100644 index faf3c7d694..0000000000 --- a/sonic_cli_gen/generator.py +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env python - -import os -import pkgutil -import jinja2 - -from sonic_cli_gen.yang_parser import YangParser - -templates_path_switch = '/usr/share/sonic/templates/sonic-cli-gen/' - - -class CliGenerator: - """ SONiC CLI generator. This class provides public API - for sonic-cli-gen python library. It can generate config, - show CLI plugins. - - Attributes: - logger: logger - """ - - def __init__(self, logger): - """ Initialize CliGenerator. """ - - self.logger = logger - - - def generate_cli_plugin( - self, - cli_group, - plugin_name, - config_db_path='configDB', - templates_path='/usr/share/sonic/templates/sonic-cli-gen/' - ): - """ Generate click CLI plugin and put it to: - /usr/local/lib//dist-packages//plugins/auto/ - """ - - parser = YangParser( - yang_model_name=plugin_name, - config_db_path=config_db_path, - allow_tbl_without_yang=True, - debug=False - ) - # yang_dict will be used as an input for templates located in - # /usr/share/sonic/templates/sonic-cli-gen/ - yang_dict = parser.parse_yang_model() - - loader = jinja2.FileSystemLoader(templates_path) - j2_env = jinja2.Environment(loader=loader) - template = j2_env.get_template(cli_group + '.py.j2') - - plugin_path = get_cli_plugin_path(cli_group, plugin_name + '_yang.py') - - with open(plugin_path, 'w') as plugin_py: - plugin_py.write(template.render(yang_dict)) - self.logger.info(' Auto-generation successful! Location: {}'.format(plugin_path)) - - - def remove_cli_plugin(self, cli_group, plugin_name): - """ Remove CLI plugin from directory: - /usr/local/lib//dist-packages//plugins/auto/ - """ - - plugin_path = get_cli_plugin_path(cli_group, plugin_name + '_yang.py') - - if os.path.exists(plugin_path): - os.remove(plugin_path) - self.logger.info(' {} was removed.'.format(plugin_path)) - else: - self.logger.info(' Path {} doest NOT exist!'.format(plugin_path)) - - -def get_cli_plugin_path(command, plugin_name): - pkg_loader = pkgutil.get_loader(f'{command}.plugins.auto') - if pkg_loader is None: - raise Exception(f'Failed to get plugins path for {command} CLI') - plugins_pkg_path = os.path.dirname(pkg_loader.path) - - return os.path.join(plugins_pkg_path, plugin_name) - diff --git a/sonic_cli_gen/main.py b/sonic_cli_gen/main.py deleted file mode 100644 index bfcd301aed..0000000000 --- a/sonic_cli_gen/main.py +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env python - -import sys -import click -import logging -from sonic_cli_gen.generator import CliGenerator - -logger = logging.getLogger('sonic-cli-gen') -logging.basicConfig(stream=sys.stdout, level=logging.INFO) - - -@click.group() -@click.pass_context -def cli(ctx): - """ SONiC CLI Auto-generator tool.\r - Generate click CLI plugin for 'config' or 'show' CLI groups.\r - CLI plugin will be generated from the YANG model, which should be in:\r\n - /usr/local/yang-models/ \n - Generated CLI plugin will be placed in: \r\n - /usr/local/lib/python3.7/dist-packages//plugins/auto/ - """ - - context = { - 'gen': CliGenerator(logger) - } - ctx.obj = context - - -@cli.command() -@click.argument('cli_group', type=click.Choice(['config', 'show'])) -@click.argument('yang_model_name', type=click.STRING) -@click.pass_context -def generate(ctx, cli_group, yang_model_name): - """ Generate click CLI plugin. """ - - ctx.obj['gen'].generate_cli_plugin(cli_group, yang_model_name) - - -@cli.command() -@click.argument('cli_group', type=click.Choice(['config', 'show'])) -@click.argument('yang_model_name', type=click.STRING) -@click.pass_context -def remove(ctx, cli_group, yang_model_name): - """ Remove generated click CLI plugin from. """ - - ctx.obj['gen'].remove_cli_plugin(cli_group, yang_model_name) - - -if __name__ == '__main__': - cli() - diff --git a/sonic_cli_gen/yang_parser.py b/sonic_cli_gen/yang_parser.py deleted file mode 100644 index df0382536f..0000000000 --- a/sonic_cli_gen/yang_parser.py +++ /dev/null @@ -1,679 +0,0 @@ -#!/usr/bin/env python - -from collections import OrderedDict -from config.config_mgmt import ConfigMgmt -from typing import List, Dict - -yang_guidelines_link = 'https://github.com/Azure/SONiC/blob/master/doc/mgmt/SONiC_YANG_Model_Guidelines.md' - - -class YangParser: - """ YANG model parser - - Attributes: - yang_model_name: Name of the YANG model file - conf_mgmt: Instance of Config Mgmt class to - help parse YANG models - y_module: Reference to 'module' entity - from YANG model file - y_top_level_container: Reference to top level 'container' - entity from YANG model file - y_table_containers: Reference to 'container' entities - from YANG model file that represent Config DB tables - yang_2_dict: dictionary created from YANG model file that - represent Config DB schema. - - Below the 'yang_2_dict' obj in case if YANG model has a 'list' entity: - { - 'tables': [{ - 'name': 'value', - 'description': 'value', - 'dynamic-objects': [ - 'name': 'value', - 'description': 'value, - 'attrs': [ - { - 'name': 'value', - 'description': 'value', - 'is-leaf-list': False, - 'is-mandatory': False, - 'group': 'value' - } - ... - ], - 'keys': [ - { - 'name': 'ACL_TABLE_NAME', - 'description': 'value' - } - ... - ] - ], - }] - } - In case if YANG model does NOT have a 'list' entity, - it has the same structure as above, but 'dynamic-objects' - changed to 'static-objects' and have no 'keys' - """ - - def __init__(self, - yang_model_name, - config_db_path, - allow_tbl_without_yang, - debug): - self.yang_model_name = yang_model_name - self.conf_mgmt = None - self.y_module = None - self.y_top_level_container = None - self.y_table_containers = None - self.yang_2_dict = dict() - - try: - self.conf_mgmt = ConfigMgmt(config_db_path, - debug, - allow_tbl_without_yang) - except Exception as e: - raise Exception("Failed to load the {} class".format(str(e))) - - def _init_yang_module_and_containers(self): - """ Initialize inner class variables: - self.y_module - self.y_top_level_container - self.y_table_containers - - Raises: - Exception: if YANG model is invalid or NOT exist - """ - - self.y_module = self._find_yang_model_in_yjson_obj() - - if self.y_module is None: - raise Exception('The YANG model {} is NOT exist'.format(self.yang_model_name)) - - if self.y_module.get('container') is None: - raise Exception('The YANG model {} does NOT have\ - "top level container" element\ - Please follow the SONiC YANG model guidelines:\ - \n{}'.format(self.yang_model_name, yang_guidelines_link)) - self.y_top_level_container = self.y_module.get('container') - - if self.y_top_level_container.get('container') is None: - raise Exception('The YANG model {} does NOT have "container"\ - element after "top level container"\ - Please follow the SONiC YANG model guidelines:\ - \n{}'.format(self.yang_model_name, yang_guidelines_link)) - self.y_table_containers = self.y_top_level_container.get('container') - - def _find_yang_model_in_yjson_obj(self) -> OrderedDict: - """ Find provided YANG model inside the yJson object, - the yJson object contain all yang-models - parsed from directory - /usr/local/yang-models - - Returns: - reference to yang_model_name - """ - - for yang_model in self.conf_mgmt.sy.yJson: - if yang_model.get('module').get('@name') == self.yang_model_name: - return yang_model.get('module') - - def parse_yang_model(self) -> dict: - """ Parse provided YANG model and save - the output to self.yang_2_dict object - - Returns: - parsed YANG model in dictionary format - """ - - self._init_yang_module_and_containers() - self.yang_2_dict['tables'] = list() - - # determine how many (1 or more) containers a YANG model - # has after the 'top level' container - # 'table' container goes after the 'top level' container - self.yang_2_dict['tables'] += list_handler(self.y_table_containers, - lambda e: on_table_container(self.y_module, e, self.conf_mgmt)) - - return self.yang_2_dict - - -# ------------------------------HANDLERS-------------------------------- # - -def list_handler(y_entity, callback) -> List[Dict]: - """ Determine if the type of entity is a list, - if so - call the callback for every list element - """ - - if isinstance(y_entity, list): - return [callback(e) for e in y_entity] - else: - return [callback(y_entity)] - - -def on_table_container(y_module: OrderedDict, - tbl_container: OrderedDict, - conf_mgmt: ConfigMgmt) -> dict: - """ Parse 'table' container, - 'table' container goes after 'top level' container - - Args: - y_module: reference to 'module' - tbl_container: reference to 'table' container - conf_mgmt: reference to ConfigMgmt class instance, - it have yJson object which contain all parsed YANG models - Returns: - element for self.yang_2_dict['tables'] - """ - y2d_elem = { - 'name': tbl_container.get('@name'), - 'description': get_description(tbl_container) - } - - # determine if 'table container' has a 'list' entity - if tbl_container.get('list') is None: - y2d_elem['static-objects'] = list() - - # 'object' container goes after the 'table' container - # 'object' container have 2 types - list (like sonic-flex_counter.yang) - # and NOT list (like sonic-device_metadata.yang) - y2d_elem['static-objects'] += list_handler(tbl_container.get('container'), - lambda e: on_object_entity(y_module, e, conf_mgmt, is_list=False)) - else: - y2d_elem['dynamic-objects'] = list() - - # 'container' can have more than 1 'list' entity - y2d_elem['dynamic-objects'] += list_handler(tbl_container.get('list'), - lambda e: on_object_entity(y_module, e, conf_mgmt, is_list=True)) - - # move 'keys' elements from 'attrs' to 'keys' - change_dyn_obj_struct(y2d_elem['dynamic-objects']) - - return y2d_elem - - -def on_object_entity(y_module: OrderedDict, - y_entity: OrderedDict, - conf_mgmt: ConfigMgmt, - is_list: bool) -> dict: - """ Parse a 'object' entity, it could be a 'container' or a 'list' - 'Object' entity represent OBJECT in Config DB schema: - { - "TABLE": { - "OBJECT": { - "attr": "value" - } - } - } - - Args: - y_module: reference to 'module' - y_entity: reference to 'object' entity - conf_mgmt: reference to ConfigMgmt class instance, - it have yJson object which contain all parsed YANG models - is_list: boolean flag to determine if a 'list' was passed - Returns: - element for y2d_elem['static-objects'] OR y2d_elem['dynamic-objects'] - """ - - if y_entity is None: - return {} - - obj_elem = { - 'name': y_entity.get('@name'), - 'description': get_description(y_entity), - 'attrs': list() - } - - if is_list: - obj_elem['keys'] = get_list_keys(y_entity) - - attrs_list = list() - # grouping_name is empty because 'grouping' is not used so far - attrs_list.extend(get_leafs(y_entity, grouping_name='')) - attrs_list.extend(get_leaf_lists(y_entity, grouping_name='')) - attrs_list.extend(get_choices(y_module, y_entity, conf_mgmt, grouping_name='')) - attrs_list.extend(get_uses(y_module, y_entity, conf_mgmt)) - - obj_elem['attrs'] = attrs_list - - return obj_elem - - -def on_uses(y_module: OrderedDict, - y_uses, - conf_mgmt: ConfigMgmt) -> list: - """ Parse a YANG 'uses' entities - 'uses' referring to 'grouping' YANG entity - - Args: - y_module: reference to 'module' - y_uses: reference to 'uses' - conf_mgmt: reference to ConfigMgmt class instance, - it have yJson object which contain all parsed YANG model - Returns: - element for obj_elem['attrs'], 'attrs' contain a parsed 'leafs' - """ - - ret_attrs = list() - y_grouping = get_all_grouping(y_module, y_uses, conf_mgmt) - # trim prefixes in order to the next checks - trim_uses_prefixes(y_uses) - - # TODO: 'refine' support - for group in y_grouping: - if isinstance(y_uses, list): - for use in y_uses: - if group.get('@name') == use.get('@name'): - ret_attrs.extend(get_leafs(group, group.get('@name'))) - ret_attrs.extend(get_leaf_lists(group, group.get('@name'))) - ret_attrs.extend(get_choices(y_module, group, conf_mgmt, group.get('@name'))) - else: - if group.get('@name') == y_uses.get('@name'): - ret_attrs.extend(get_leafs(group, group.get('@name'))) - ret_attrs.extend(get_leaf_lists(group, group.get('@name'))) - ret_attrs.extend(get_choices(y_module, group, conf_mgmt, group.get('@name'))) - - return ret_attrs - - -def on_choices(y_module: OrderedDict, - y_choices, - conf_mgmt: ConfigMgmt, - grouping_name: str) -> list: - """ Parse a YANG 'choice' entities - - Args: - y_module: reference to 'module' - y_choices: reference to 'choice' element - conf_mgmt: reference to ConfigMgmt class instance, - it have yJson object which contain all parsed YANG model - grouping_name: if YANG entity contain 'uses', this arg represent 'grouping' name - Returns: - element for obj_elem['attrs'], 'attrs' contain a parsed 'leafs' - """ - - ret_attrs = list() - - # the YANG model can have multiple 'choice' entities - # inside a 'container' or 'list' - if isinstance(y_choices, list): - for choice in y_choices: - attrs = on_choice_cases(y_module, choice.get('case'), - conf_mgmt, grouping_name) - ret_attrs.extend(attrs) - else: - ret_attrs = on_choice_cases(y_module, y_choices.get('case'), - conf_mgmt, grouping_name) - - return ret_attrs - - -def on_choice_cases(y_module: OrderedDict, - y_cases, - conf_mgmt: ConfigMgmt, - grouping_name: str) -> list: - """ Parse a single YANG 'case' entity from the 'choice' entity. - The 'case' element can has inside - 'leaf', 'leaf-list', 'uses' - - Args: - y_module: reference to 'module' - y_cases: reference to 'case' - conf_mgmt: reference to ConfigMgmt class instance, - it have yJson object which contain all - parsed YANG model - grouping_name: if YANG entity contain 'uses', - this argument represent 'grouping' name - Returns: - element for the obj_elem['attrs'], the 'attrs' - contain a parsed 'leafs' - """ - - ret_attrs = list() - - if isinstance(y_cases, list): - for case in y_cases: - ret_attrs.extend(get_leafs(case, grouping_name)) - ret_attrs.extend(get_leaf_lists(case, grouping_name)) - ret_attrs.extend(get_uses(y_module, case, conf_mgmt)) - else: - ret_attrs.extend(get_leafs(y_cases, grouping_name)) - ret_attrs.extend(get_leaf_lists(y_cases, grouping_name)) - ret_attrs.extend(get_uses(y_module, y_cases, conf_mgmt)) - - return ret_attrs - - -def on_leafs(y_leafs, - grouping_name: str, - is_leaf_list: bool) -> list: - """ Parse all the 'leaf' or 'leaf-list' elements - - Args: - y_leafs: reference to all 'leaf' elements - grouping_name: if YANG entity contain 'uses', - this argument represent the 'grouping' name - is_leaf_list: boolean to determine if a 'leaf-list' - was passed as 'y_leafs' argument - Returns: - list of parsed 'leaf' elements - """ - - ret_attrs = list() - # The YANG 'container' entity may have only 1 'leaf' - # element OR a list of 'leaf' elements - ret_attrs += list_handler(y_leafs, lambda e: on_leaf(e, grouping_name, is_leaf_list)) - - return ret_attrs - - -def on_leaf(leaf: OrderedDict, - grouping_name: str, - is_leaf_list: bool) -> dict: - """ Parse a single 'leaf' element - - Args: - leaf: reference to a 'leaf' entity - grouping_name: if YANG entity contain 'uses', - this argument represent 'grouping' name - is_leaf_list: boolean to determine if 'leaf-list' - was passed in 'y_leafs' argument - Returns: - parsed 'leaf' element - """ - - attr = {'name': leaf.get('@name'), - 'description': get_description(leaf), - 'is-leaf-list': is_leaf_list, - 'is-mandatory': get_mandatory(leaf), - 'group': grouping_name} - - return attr - - -# ----------------------GETERS------------------------- # - -def get_mandatory(y_leaf: OrderedDict) -> bool: - """ Parse the 'mandatory' statement for a 'leaf' - - Args: - y_leaf: reference to a 'leaf' entity - Returns: - 'leaf' 'mandatory' value - """ - - if y_leaf.get('mandatory') is not None: - return True - - return False - - -def get_description(y_entity: OrderedDict) -> str: - """ Parse the 'description' entity from any YANG element - - Args: - y_entity: reference to YANG 'container' OR 'list' OR 'leaf' ... - Returns: - text of the 'description' - """ - - if y_entity.get('description') is not None: - return y_entity.get('description').get('text') - else: - return '' - - -def get_leafs(y_entity: OrderedDict, - grouping_name: str) -> list: - """ Check if the YANG entity have 'leafs', if so call handler - - Args: - y_entity: reference YANG 'container' or 'list' - or 'choice' or 'uses' - grouping_name: if YANG entity contain 'uses', - this argument represent 'grouping' name - Returns: - list of parsed 'leaf' elements - """ - - if y_entity.get('leaf') is not None: - return on_leafs(y_entity.get('leaf'), grouping_name, is_leaf_list=False) - - return [] - - -def get_leaf_lists(y_entity: OrderedDict, - grouping_name: str) -> list: - """ Check if the YANG entity have 'leaf-list', if so call handler - - Args: - y_entity: reference YANG 'container' or 'list' - or 'choice' or 'uses' - grouping_name: if YANG entity contain 'uses', - this argument represent 'grouping' name - Returns: - list of parsed 'leaf-list' elements - """ - - if y_entity.get('leaf-list') is not None: - return on_leafs(y_entity.get('leaf-list'), grouping_name, is_leaf_list=True) - - return [] - - -def get_choices(y_module: OrderedDict, - y_entity: OrderedDict, - conf_mgmt: ConfigMgmt, - grouping_name: str) -> list: - """ Check if the YANG entity have 'choice', if so call handler - - Args: - y_module: reference to 'module' - y_entity: reference YANG 'container' or 'list' - or 'choice' or 'uses' - conf_mgmt: reference to ConfigMgmt class instance, - it have yJson object which contain all parsed YANG model - grouping_name: if YANG entity contain 'uses', - this argument represent 'grouping' name - Returns: - list of parsed elements inside 'choice' - """ - - if y_entity.get('choice') is not None: - return on_choices(y_module, y_entity.get('choice'), conf_mgmt, grouping_name) - - return [] - - -def get_uses(y_module: OrderedDict, - y_entity: OrderedDict, - conf_mgmt: ConfigMgmt) -> list: - """ Check if the YANG entity have 'uses', if so call handler - - Args: - y_module: reference to 'module' - y_entity: reference YANG 'container' or 'list' - or 'choice' or 'uses' - conf_mgmt: reference to ConfigMgmt class instance, - it have yJson object which contain all parsed YANG model - Returns: - list of parsed elements inside 'grouping' - that referenced by 'uses' - """ - - if y_entity.get('uses') is not None: - return on_uses(y_module, y_entity.get('uses'), conf_mgmt) - - return [] - - -def get_all_grouping(y_module: OrderedDict, - y_uses: OrderedDict, - conf_mgmt: ConfigMgmt) -> list: - """ Get all the 'grouping' entities that was referenced - by 'uses' in current YANG model - - Args: - y_module: reference to 'module' - y_entity: reference to 'uses' - conf_mgmt: reference to ConfigMgmt class instance, - it have yJson object which contain all parsed YANG model - Returns: - list of 'grouping' elements - """ - - ret_grouping = list() - # prefix_list needed to find what YANG model was imported - prefix_list = get_import_prefixes(y_uses) - - # in case if 'grouping' located in the same YANG model - local_grouping = y_module.get('grouping') - if local_grouping is not None: - if isinstance(local_grouping, list): - ret_grouping.extend(local_grouping) - else: - ret_grouping.append(local_grouping) - - # if prefix_list is NOT empty it means that 'grouping' - # was imported from another YANG model - if prefix_list != []: - for prefix in prefix_list: - y_import = y_module.get('import') - if isinstance(y_import, list): - for _import in y_import: - if _import.get('prefix').get('@value') == prefix: - ret_grouping.extend(get_grouping_from_another_yang_model(_import.get('@module'), conf_mgmt)) - else: - if y_import.get('prefix').get('@value') == prefix: - ret_grouping.extend(get_grouping_from_another_yang_model(y_import.get('@module'), conf_mgmt)) - - return ret_grouping - - -def get_grouping_from_another_yang_model(yang_model_name: str, - conf_mgmt) -> list: - """ Get the YANG 'grouping' entity - - Args: - yang_model_name - YANG model to search - conf_mgmt - reference to ConfigMgmt class instance, - it have yJson object which contain all parsed YANG models - - Returns: - list of 'grouping' entities - """ - - ret_grouping = list() - - for yang_model in conf_mgmt.sy.yJson: - if (yang_model.get('module').get('@name') == yang_model_name): - grouping = yang_model.get('module').get('grouping') - if isinstance(grouping, list): - ret_grouping.extend(grouping) - else: - ret_grouping.append(grouping) - - return ret_grouping - - -def get_import_prefixes(y_uses: OrderedDict) -> list: - """ Parse 'import prefix' of YANG 'uses' entity - Example: - { - uses stypes:endpoint; - } - 'stypes' - prefix of imported YANG module. - 'endpoint' - YANG 'grouping' entity name - - Args: - y_uses: refrence to YANG 'uses' - Returns: - list of parsed prefixes - """ - - ret_prefixes = list() - - if isinstance(y_uses, list): - for use in y_uses: - prefix = use.get('@name').split(':')[0] - if prefix != use.get('@name'): - ret_prefixes.append(prefix) - else: - prefix = y_uses.get('@name').split(':')[0] - if prefix != y_uses.get('@name'): - ret_prefixes.append(prefix) - - return ret_prefixes - - -def trim_uses_prefixes(y_uses) -> list: - """ Trim prefixes from the 'uses' YANG entities. - If the YANG 'grouping' was imported from another - YANG file, it use the 'prefix' before the 'grouping' name: - { - uses sgrop:endpoint; - } - Where 'sgrop' = 'prefix'; 'endpoint' = 'grouping' name. - - Args: - y_uses: reference to 'uses' - - Returns: - list of 'uses' without 'prefixes' - """ - - prefixes = get_import_prefixes(y_uses) - - for prefix in prefixes: - if isinstance(y_uses, list): - for use in y_uses: - if prefix in use.get('@name'): - use['@name'] = use.get('@name').split(':')[1] - else: - if prefix in y_uses.get('@name'): - y_uses['@name'] = y_uses.get('@name').split(':')[1] - - -def get_list_keys(y_list: OrderedDict) -> list: - """ Parse YANG the 'key' entity. - If YANG model has a 'list' entity, inside the 'list' - there is 'key' entity. The 'key' - whitespace - separeted list of 'leafs' - - Args: - y_list: reference to the 'list' - Returns: - list of parsed keys - """ - - ret_list = list() - - keys = y_list.get('key').get('@value').split() - for k in keys: - key = {'name': k} - ret_list.append(key) - - return ret_list - - -def change_dyn_obj_struct(dynamic_objects: list): - """ Rearrange self.yang_2_dict['dynamic_objects'] structure. - If YANG model have a 'list' entity - inside the 'list' - it has 'key' entity. The 'key' entity it is whitespace - separeted list of 'leafs', those 'leafs' was parsed by - 'on_leaf()' function and placed under 'attrs' in - self.yang_2_dict['dynamic_objects'] need to move 'leafs' - from 'attrs' and put them into 'keys' section of - self.yang_2_dict['dynamic_objects'] - - Args: - dynamic_objects: reference to self.yang_2_dict['dynamic_objects'] - """ - - for obj in dynamic_objects: - for key in obj.get('keys'): - for attr in obj.get('attrs'): - if key.get('name') == attr.get('name'): - key['description'] = attr.get('description') - obj['attrs'].remove(attr) - break - diff --git a/tests/cli_autogen_input/autogen_test/show_cmd_output.py b/tests/cli_autogen_input/autogen_test/show_cmd_output.py deleted file mode 100644 index 19c02c7783..0000000000 --- a/tests/cli_autogen_input/autogen_test/show_cmd_output.py +++ /dev/null @@ -1,81 +0,0 @@ -""" -This module are holding correct output for the show command for cli_autogen_test.py -""" - - -show_device_metadata_localhost="""\ -HWSKU DEFAULT BGP STATUS DOCKER ROUTING CONFIG MODE HOSTNAME PLATFORM MAC DEFAULT PFCWD STATUS BGP ASN DEPLOYMENT ID TYPE BUFFER MODEL FRR MGMT FRAMEWORK CONFIG ------------ -------------------- ---------------------------- ---------- ---------------------- ----------------- ---------------------- --------- --------------- --------- -------------- --------------------------- -ACS-MSN2100 up N/A r-sonic-01 x86_64-mlnx_msn2100-r0 ff:ff:ff:ff:ff:00 disable N/A N/A ToRRouter traditional N/A -""" - - -show_device_metadata_localhost_changed_buffer_model="""\ -HWSKU DEFAULT BGP STATUS DOCKER ROUTING CONFIG MODE HOSTNAME PLATFORM MAC DEFAULT PFCWD STATUS BGP ASN DEPLOYMENT ID TYPE BUFFER MODEL FRR MGMT FRAMEWORK CONFIG ------------ -------------------- ---------------------------- ---------- ---------------------- ----------------- ---------------------- --------- --------------- --------- -------------- --------------------------- -ACS-MSN2100 up N/A r-sonic-01 x86_64-mlnx_msn2100-r0 ff:ff:ff:ff:ff:00 disable N/A N/A ToRRouter dynamic N/A -""" - - -show_device_neighbor="""\ -PEER NAME NAME MGMT ADDR LOCAL PORT PORT TYPE ------------ -------- ----------- ------------ ------ ------ -Ethernet0 Servers 10.217.0.1 Ethernet0 eth0 type -Ethernet4 Servers0 10.217.0.2 Ethernet4 eth1 type -""" - - -show_device_neighbor_added="""\ -PEER NAME NAME MGMT ADDR LOCAL PORT PORT TYPE ------------ -------- ----------- ------------ ------ ------ -Ethernet0 Servers 10.217.0.1 Ethernet0 eth0 type -Ethernet4 Servers0 10.217.0.2 Ethernet4 eth1 type -Ethernet8 Servers1 10.217.0.3 Ethernet8 eth2 type -""" - - -show_device_neighbor_deleted="""\ -PEER NAME NAME MGMT ADDR LOCAL PORT PORT TYPE ------------ -------- ----------- ------------ ------ ------ -Ethernet4 Servers0 10.217.0.2 Ethernet4 eth1 type -""" - - -show_device_neighbor_updated_mgmt_addr="""\ -PEER NAME NAME MGMT ADDR LOCAL PORT PORT TYPE ------------ -------- ----------- ------------ ------ ------ -Ethernet0 Servers 10.217.0.5 Ethernet0 eth0 type -Ethernet4 Servers0 10.217.0.2 Ethernet4 eth1 type -""" - - -show_device_neighbor_updated_name="""\ -PEER NAME NAME MGMT ADDR LOCAL PORT PORT TYPE ------------ -------- ----------- ------------ ------ ------ -Ethernet0 Servers1 10.217.0.1 Ethernet0 eth0 type -Ethernet4 Servers0 10.217.0.2 Ethernet4 eth1 type -""" - - -show_device_neighbor_updated_local_port="""\ -PEER NAME NAME MGMT ADDR LOCAL PORT PORT TYPE ------------ -------- ----------- ------------ ------ ------ -Ethernet0 Servers 10.217.0.1 Ethernet12 eth0 type -Ethernet4 Servers0 10.217.0.2 Ethernet4 eth1 type -""" - - -show_device_neighbor_updated_port="""\ -PEER NAME NAME MGMT ADDR LOCAL PORT PORT TYPE ------------ -------- ----------- ------------ ------ ------ -Ethernet0 Servers 10.217.0.1 Ethernet0 eth2 type -Ethernet4 Servers0 10.217.0.2 Ethernet4 eth1 type -""" - - -show_device_neighbor_updated_type="""\ -PEER NAME NAME MGMT ADDR LOCAL PORT PORT TYPE ------------ -------- ----------- ------------ ------ ------ -Ethernet0 Servers 10.217.0.1 Ethernet0 eth0 type2 -Ethernet4 Servers0 10.217.0.2 Ethernet4 eth1 type -""" diff --git a/tests/cli_autogen_input/autogen_test/sonic-device_metadata.yang b/tests/cli_autogen_input/autogen_test/sonic-device_metadata.yang deleted file mode 100644 index 400cbf3bcd..0000000000 --- a/tests/cli_autogen_input/autogen_test/sonic-device_metadata.yang +++ /dev/null @@ -1,123 +0,0 @@ -module sonic-device_metadata { - - yang-version 1.1; - - namespace "http://github.com/Azure/sonic-device_metadata"; - prefix device_metadata; - - import ietf-yang-types { - prefix yang; - } - - import ietf-inet-types { - prefix inet; - } - - import sonic-types { - prefix stypes; - revision-date 2019-07-01; - } - - description "DEVICE_METADATA YANG Module for SONiC OS"; - - revision 2021-02-27 { - description "Added frr_mgmt_framework_config field to handle BGP - config DB schema events to configure FRR protocols."; - } - - revision 2020-04-10 { - description "First Revision"; - } - - container sonic-device_metadata { - - container DEVICE_METADATA { - - description "DEVICE_METADATA part of config_db.json"; - - container localhost{ - - leaf hwsku { - type stypes:hwsku; - } - - leaf default_bgp_status { - type enumeration { - enum up; - enum down; - } - default up; - } - - leaf docker_routing_config_mode { - type string { - pattern "unified|split|separated"; - } - default "unified"; - } - - leaf hostname { - type string { - length 1..255; - } - } - - leaf platform { - type string { - length 1..255; - } - } - - leaf mac { - type yang:mac-address; - } - - leaf default_pfcwd_status { - type enumeration { - enum disable; - enum enable; - } - default disable; - } - - leaf bgp_asn { - type inet:as-number; - } - - leaf deployment_id { - type uint32; - } - - leaf type { - type string { - length 1..255; - pattern "ToRRouter|LeafRouter|SpineChassisFrontendRouter|ChassisBackendRouter|ASIC"; - } - } - - leaf buffer_model { - description "This leaf is added for dynamic buffer calculation. - The dynamic model represents the model in which the buffer configurations, - like the headroom sizes and buffer pool sizes, are dynamically calculated based - on the ports' speed, cable length, and MTU. This model is used by Mellanox so far. - The traditional model represents the model in which all the buffer configurations - are statically configured in CONFIG_DB tables. This is the default model used by all other vendors"; - type string { - pattern "dynamic|traditional"; - } - } - - leaf frr_mgmt_framework_config { - type boolean; - description "FRR configurations are handled by sonic-frr-mgmt-framework module when set to true, - otherwise, sonic-bgpcfgd handles the FRR configurations based on the predefined templates."; - default "false"; - } - } - /* end of container localhost */ - } - /* end of container DEVICE_METADATA */ - } - /* end of top level container */ -} -/* end of module sonic-device_metadata */ diff --git a/tests/cli_autogen_input/autogen_test/sonic-device_neighbor.yang b/tests/cli_autogen_input/autogen_test/sonic-device_neighbor.yang deleted file mode 100644 index e1c745dd9a..0000000000 --- a/tests/cli_autogen_input/autogen_test/sonic-device_neighbor.yang +++ /dev/null @@ -1,78 +0,0 @@ -module sonic-device_neighbor { - - yang-version 1.1; - - namespace "http://github.com/Azure/sonic-device_neighbor"; - prefix device_neighbor; - - import ietf-inet-types { - prefix inet; - } - - import sonic-extension { - prefix ext; - revision-date 2019-07-01; - } - - import sonic-port { - prefix port; - revision-date 2019-07-01; - } - - description "DEVICE_NEIGHBOR YANG Module for SONiC OS"; - - revision 2020-04-10 { - description "First Revision"; - } - - container sonic-device_neighbor { - - container DEVICE_NEIGHBOR { - - description "DEVICE_NEIGHBOR part of config_db.json"; - - list DEVICE_NEIGHBOR_LIST { - - key "peer_name"; - - leaf peer_name { - type string { - length 1..255; - } - } - - leaf name { - type string { - length 1..255; - } - } - - leaf mgmt_addr { - type inet:ip-address; - } - - leaf local_port { - type leafref { - path /port:sonic-port/port:PORT/port:PORT_LIST/port:name; - } - } - - leaf port { - type string { - length 1..255; - } - } - - leaf type { - type string { - length 1..255; - } - } - } - /* end of list DEVICE_NEIGHBOR_LIST */ - } - /* end of container DEVICE_NEIGHBOR */ - } - /* end of top level container */ -} -/* end of module sonic-device_neighbor */ diff --git a/tests/cli_autogen_input/cli_autogen_common.py b/tests/cli_autogen_input/cli_autogen_common.py deleted file mode 100644 index 141bceed9c..0000000000 --- a/tests/cli_autogen_input/cli_autogen_common.py +++ /dev/null @@ -1,38 +0,0 @@ -import os - -yang_models_path = '/usr/local/yang-models' - - -def move_yang_models(test_path, test_name, test_yang_models): - """ Move a test YANG models to known location """ - - for yang_model in test_yang_models: - src_path = os.path.join( - test_path, - 'cli_autogen_input', - test_name, - yang_model - ) - os.system('sudo cp {} {}'.format(src_path, yang_models_path)) - - -def remove_yang_models(test_yang_models): - """ Remove a test YANG models to known location """ - - for yang_model in test_yang_models: - yang_model_path = os.path.join(yang_models_path, yang_model) - os.system('sudo rm {}'.format(yang_model_path)) - - -def backup_yang_models(): - """ Make a copy of existing YANG models """ - - os.system('sudo cp -R {} {}'.format(yang_models_path, yang_models_path + '_backup')) - - -def restore_backup_yang_models(): - """ Restore existing YANG models from backup """ - - os.system('sudo cp {} {}'.format(yang_models_path + '_backup/*', yang_models_path)) - os.system('sudo rm -rf {}'.format(yang_models_path + '_backup')) - \ No newline at end of file diff --git a/tests/cli_autogen_input/config_db.json b/tests/cli_autogen_input/config_db.json deleted file mode 100644 index 5d8c863cec..0000000000 --- a/tests/cli_autogen_input/config_db.json +++ /dev/null @@ -1,65 +0,0 @@ -{ - "DEVICE_METADATA|localhost": { - "buffer_model": "traditional", - "default_bgp_status": "up", - "default_pfcwd_status": "disable", - "hostname": "r-sonic-01", - "hwsku": "ACS-MSN2100", - "mac": "ff:ff:ff:ff:ff:00", - "platform": "x86_64-mlnx_msn2100-r0", - "type": "ToRRouter" - }, - "PORT|Ethernet0": { - "alias": "etp1", - "description": "etp1", - "index": "0", - "lanes": "0, 1, 2, 3", - "mtu": "9100", - "pfc_asym": "off", - "speed": "100000" - }, - "PORT|Ethernet4": { - "admin_status": "up", - "alias": "etp2", - "description": "Servers0:eth0", - "index": "1", - "lanes": "4, 5, 6, 7", - "mtu": "9100", - "pfc_asym": "off", - "speed": "100000" - }, - "PORT|Ethernet8": { - "admin_status": "up", - "alias": "etp3", - "description": "Servers0:eth2", - "index": "2", - "lanes": "8, 9, 10, 11", - "mtu": "9100", - "pfc_asym": "off", - "speed": "100000" - }, - "PORT|Ethernet12": { - "admin_status": "up", - "alias": "etp4", - "description": "Servers0:eth4", - "index": "3", - "lanes": "12, 13, 14, 15", - "mtu": "9100", - "pfc_asym": "off", - "speed": "100000" - }, - "DEVICE_NEIGHBOR|Ethernet0": { - "name": "Servers", - "port": "eth0", - "mgmt_addr": "10.217.0.1", - "local_port": "Ethernet0", - "type": "type" - }, - "DEVICE_NEIGHBOR|Ethernet4": { - "name": "Servers0", - "port": "eth1", - "mgmt_addr": "10.217.0.2", - "local_port": "Ethernet4", - "type": "type" - } -} \ No newline at end of file diff --git a/tests/cli_autogen_input/yang_parser_test/assert_dictionaries.py b/tests/cli_autogen_input/yang_parser_test/assert_dictionaries.py deleted file mode 100644 index bed2a4a06a..0000000000 --- a/tests/cli_autogen_input/yang_parser_test/assert_dictionaries.py +++ /dev/null @@ -1,626 +0,0 @@ -""" -Module holding correct dictionaries for test YANG models -""" - -one_table_container = { - "tables":[ - { - "description":"TABLE_1 description", - "name":"TABLE_1", - "static-objects":[ - { - } - ] - } - ] -} - -two_table_containers = { - "tables":[ - { - "description":"TABLE_1 description", - "name":"TABLE_1", - "static-objects":[ - { - - } - ] - }, - { - "description":"TABLE_2 description", - "name":"TABLE_2", - "static-objects":[ - { - - } - ] - } - ] -} - -one_object_container = { - "tables":[ - { - "description":"TABLE_1 description", - "name":"TABLE_1", - "static-objects":[ - { - "name":"OBJECT_1", - "description":"OBJECT_1 description", - "attrs":[ - ] - } - ] - } - ] -} - -two_object_containers = { - "tables":[ - { - "description":"FIRST_TABLE description", - "name":"TABLE_1", - "static-objects":[ - { - "name":"OBJECT_1", - "description":"OBJECT_1 description", - "attrs":[ - ] - }, - { - "name":"OBJECT_2", - "description":"OBJECT_2 description", - "attrs":[ - ] - } - ] - } - ] -} - -one_list = { - "tables":[ - { - "description":"TABLE_1 description", - "name":"TABLE_1", - "dynamic-objects":[ - { - "name":"TABLE_1_LIST", - "description":"TABLE_1_LIST description", - "keys":[ - { - "name": "key_name", - "description": "", - } - ], - "attrs":[ - ] - } - ] - } - ] -} - -two_lists = { - "tables":[ - { - "description":"TABLE_1 description", - "name":"TABLE_1", - "dynamic-objects":[ - { - "name":"TABLE_1_LIST_1", - "description":"TABLE_1_LIST_1 description", - "keys":[ - { - "name": "key_name1", - "description": "", - } - ], - "attrs":[ - ] - }, - { - "name":"TABLE_1_LIST_2", - "description":"TABLE_1_LIST_2 description", - "keys":[ - { - "name": "key_name2", - "description": "", - } - ], - "attrs":[ - ] - } - ] - } - ] -} - -static_object_complex_1 = { - "tables":[ - { - "description":"TABLE_1 description", - "name":"TABLE_1", - "static-objects":[ - { - "name":"OBJECT_1", - "description":"OBJECT_1 description", - "attrs":[ - { - "name":"OBJ_1_LEAF_1", - "description": "OBJ_1_LEAF_1 description", - "is-mandatory": False, - "is-leaf-list": False, - "group": '', - }, - { - "name":"OBJ_1_LEAF_LIST_1", - "description": "", - "is-mandatory": False, - "is-leaf-list": True, - "group": '', - }, - { - "name":"OBJ_1_CHOICE_1_LEAF_1", - "description": "", - "is-mandatory": False, - "is-leaf-list": False, - "group": '', - }, - { - "name":"OBJ_1_CHOICE_1_LEAF_2", - "description": "", - "is-mandatory": False, - "is-leaf-list": False, - "group": '', - } - ] - } - ] - } - ] -} - -static_object_complex_2 = { - "tables":[ - { - "description":"TABLE_1 description", - "name":"TABLE_1", - "static-objects":[ - { - "name":"OBJECT_1", - "description":"OBJECT_1 description", - "attrs":[ - { - "name":"OBJ_1_LEAF_1", - "description": "OBJ_1_LEAF_1 description", - "is-mandatory": False, - "is-leaf-list": False, - "group": '', - }, - { - "name":"OBJ_1_LEAF_2", - "description": "OBJ_1_LEAF_2 description", - "is-mandatory": False, - "is-leaf-list": False, - "group": '', - }, - { - "name":"OBJ_1_LEAF_LIST_1", - "description": "", - "is-mandatory": False, - "is-leaf-list": True, - "group": '', - }, - { - "name":"OBJ_1_LEAF_LIST_2", - "description": "", - "is-mandatory": False, - "is-leaf-list": True, - "group": '', - }, - { - "name":"OBJ_1_CHOICE_1_LEAF_1", - "description": "", - "is-mandatory": False, - "is-leaf-list": False, - "group": '', - }, - { - "name":"OBJ_1_CHOICE_1_LEAF_2", - "description": "", - "is-mandatory": False, - "is-leaf-list": False, - "group": '', - }, - { - "name":"OBJ_1_CHOICE_2_LEAF_1", - "description": "", - "is-mandatory": False, - "is-leaf-list": False, - "group": '', - }, - { - "name":"OBJ_1_CHOICE_2_LEAF_2", - "description": "", - "is-mandatory": False, - "is-leaf-list": False, - "group": '', - }, - ] - } - ] - } - ] -} - -dynamic_object_complex_1 = { - "tables":[ - { - "description":"TABLE_1 description", - "name":"TABLE_1", - "dynamic-objects":[ - { - "name":"OBJECT_1_LIST", - "description":"OBJECT_1_LIST description", - "attrs":[ - { - "name":"OBJ_1_LEAF_1", - "description": "OBJ_1_LEAF_1 description", - "is-mandatory": False, - "is-leaf-list": False, - "group": '', - }, - { - "name":"OBJ_1_LEAF_LIST_1", - "description": "", - "is-mandatory": False, - "is-leaf-list": True, - "group": '', - }, - { - "name":"OBJ_1_CHOICE_1_LEAF_1", - "description": "", - "is-mandatory": False, - "is-leaf-list": False, - "group": '', - }, - { - "name":"OBJ_1_CHOICE_1_LEAF_2", - "description": "", - "is-mandatory": False, - "is-leaf-list": False, - "group": '', - } - ], - "keys":[ - { - "name": "KEY_LEAF_1", - "description": "KEY_LEAF_1 description", - } - ] - } - ] - } - ] -} - -dynamic_object_complex_2 = { - "tables":[ - { - "description":"TABLE_1 description", - "name":"TABLE_1", - "dynamic-objects":[ - { - "name":"OBJECT_1_LIST", - "description":"OBJECT_1_LIST description", - "attrs":[ - { - "name":"OBJ_1_LEAF_1", - "description": "OBJ_1_LEAF_1 description", - "is-mandatory": False, - "is-leaf-list": False, - "group": '', - }, - { - "name":"OBJ_1_LEAF_2", - "description": "OBJ_1_LEAF_2 description", - "is-mandatory": False, - "is-leaf-list": False, - "group": '', - }, - { - "name":"OBJ_1_LEAF_LIST_1", - "description": "", - "is-mandatory": False, - "is-leaf-list": True, - "group": '', - }, - { - "name":"OBJ_1_LEAF_LIST_2", - "description": "", - "is-mandatory": False, - "is-leaf-list": True, - "group": '', - }, - { - "name":"OBJ_1_CHOICE_1_LEAF_1", - "description": "", - "is-mandatory": False, - "is-leaf-list": False, - "group": '', - }, - { - "name":"OBJ_1_CHOICE_1_LEAF_2", - "description": "", - "is-mandatory": False, - "is-leaf-list": False, - "group": '', - }, - { - "name":"OBJ_1_CHOICE_2_LEAF_1", - "description": "", - "is-mandatory": False, - "is-leaf-list": False, - "group": '', - }, - { - "name":"OBJ_1_CHOICE_2_LEAF_2", - "description": "", - "is-mandatory": False, - "is-leaf-list": False, - "group": '', - } - ], - "keys":[ - { - "name": "KEY_LEAF_1", - "description": "KEY_LEAF_1 description", - }, - { - "name": "KEY_LEAF_2", - "description": "KEY_LEAF_2 description", - } - ] - } - ] - } - ] -} - -choice_complex = { - "tables":[ - { - "description":"TABLE_1 description", - "name":"TABLE_1", - "static-objects":[ - { - "name":"OBJECT_1", - "description":"OBJECT_1 description", - "attrs":[ - { - "name":"LEAF_1", - "description": "", - "is-mandatory": False, - "is-leaf-list": False, - "group": '', - }, - { - "name":"LEAF_LIST_1", - "description": "", - "is-mandatory": False, - "is-leaf-list": True, - "group": '', - }, - { - "name":"GR_1_LEAF_1", - "description": "", - "is-mandatory": False, - "is-leaf-list": False, - "group": "GR_1", - }, - { - "name":"GR_1_LEAF_2", - "description": "", - "is-mandatory": False, - "is-leaf-list": False, - "group": 'GR_1', - }, - { - "name":"LEAF_2", - "description": "", - "is-mandatory": False, - "is-leaf-list": False, - "group": '', - }, - { - "name":"LEAF_3", - "description": "", - "is-mandatory": False, - "is-leaf-list": False, - "group": '', - }, - { - "name":"LEAF_LIST_2", - "description": "", - "is-mandatory": False, - "is-leaf-list": True, - "group": '', - }, - { - "name":"LEAF_LIST_3", - "description": "", - "is-mandatory": False, - "is-leaf-list": True, - "group": '', - }, - { - "name":"GR_5_LEAF_1", - "description": "", - "is-mandatory": False, - "is-leaf-list": False, - "group": 'GR_5', - }, - { - "name":"GR_5_LEAF_2", - "description": "", - "is-mandatory": False, - "is-leaf-list": False, - "group": 'GR_5', - }, - { - "name":"GR_2_LEAF_1", - "description": "", - "is-mandatory": False, - "is-leaf-list": False, - "group": 'GR_2', - }, - { - "name":"GR_2_LEAF_2", - "description": "", - "is-mandatory": False, - "is-leaf-list": False, - "group": 'GR_2', - }, - { - "name":"GR_3_LEAF_1", - "description": "", - "is-mandatory": False, - "is-leaf-list": False, - "group": 'GR_3', - }, - { - "name":"GR_3_LEAF_2", - "description": "", - "is-mandatory": False, - "is-leaf-list": False, - "group": 'GR_3', - }, - ] - } - ] - } - ] -} - -grouping_complex = { - "tables":[ - { - "description":"TABLE_1 description", - "name":"TABLE_1", - "static-objects":[ - { - "name":"OBJECT_1", - "description":"OBJECT_1 description", - "attrs":[ - { - "name":"GR_1_LEAF_1", - "description": "", - "is-mandatory": False, - "is-leaf-list": False, - "group": "GR_1", - }, - { - "name":"GR_1_LEAF_2", - "description": "", - "is-mandatory": False, - "is-leaf-list": False, - "group": 'GR_1', - }, - ] - }, - { - "name":"OBJECT_2", - "description":"OBJECT_2 description", - "attrs":[ - { - "name":"GR_5_LEAF_1", - "description": "", - "is-mandatory": False, - "is-leaf-list": False, - "group": "GR_5", - }, - { - "name":"GR_5_LEAF_LIST_1", - "description": "", - "is-mandatory": False, - "is-leaf-list": True, - "group": "GR_5", - }, - { - "name":"GR_6_LEAF_1", - "description": "", - "is-mandatory": False, - "is-leaf-list": False, - "group": "GR_6", - }, - { - "name":"GR_6_LEAF_2", - "description": "", - "is-mandatory": False, - "is-leaf-list": False, - "group": "GR_6", - }, - { - "name":"GR_6_CASE_1_LEAF_1", - "description": "", - "is-mandatory": False, - "is-leaf-list": False, - "group": "GR_6", - }, - { - "name":"GR_6_CASE_1_LEAF_LIST_1", - "description": "", - "is-mandatory": False, - "is-leaf-list": True, - "group": "GR_6", - }, - { - "name":"GR_6_CASE_2_LEAF_1", - "description": "", - "is-mandatory": False, - "is-leaf-list": False, - "group": "GR_6", - }, - { - "name":"GR_6_CASE_2_LEAF_2", - "description": "", - "is-mandatory": False, - "is-leaf-list": False, - "group": "GR_6", - }, - { - "name":"GR_6_CASE_2_LEAF_LIST_1", - "description": "", - "is-mandatory": False, - "is-leaf-list": True, - "group": "GR_6", - }, - { - "name":"GR_6_CASE_2_LEAF_LIST_2", - "description": "", - "is-mandatory": False, - "is-leaf-list": True, - "group": "GR_6", - }, - { - "name":"GR_4_LEAF_1", - "description": "", - "is-mandatory": False, - "is-leaf-list": False, - "group": "GR_4", - }, - { - "name":"GR_4_LEAF_2", - "description": "", - "is-mandatory": False, - "is-leaf-list": False, - "group": "GR_4", - }, - ] - } - ] - } - ] -} - diff --git a/tests/cli_autogen_input/yang_parser_test/sonic-1-list.yang b/tests/cli_autogen_input/yang_parser_test/sonic-1-list.yang deleted file mode 100644 index bc8603add4..0000000000 --- a/tests/cli_autogen_input/yang_parser_test/sonic-1-list.yang +++ /dev/null @@ -1,29 +0,0 @@ -module sonic-1-list { - - yang-version 1.1; - - namespace "http://github.com/Azure/s-1-list"; - prefix s-1-list; - - container sonic-1-list { - /* sonic-1-list - top level container */ - - container TABLE_1 { - /* TABLE_1 - table container */ - - description "TABLE_1 description"; - - list TABLE_1_LIST { - /* TABLE_1 - object container */ - - description "TABLE_1_LIST description"; - - key "key_name"; - - leaf key_name { - type string; - } - } - } - } -} diff --git a/tests/cli_autogen_input/yang_parser_test/sonic-1-object-container.yang b/tests/cli_autogen_input/yang_parser_test/sonic-1-object-container.yang deleted file mode 100644 index 8d19979157..0000000000 --- a/tests/cli_autogen_input/yang_parser_test/sonic-1-object-container.yang +++ /dev/null @@ -1,23 +0,0 @@ -module sonic-1-object-container { - - yang-version 1.1; - - namespace "http://github.com/Azure/s-1-object"; - prefix s-1-object; - - container sonic-1-object-container { - /* sonic-1-object-container - top level container */ - - container TABLE_1 { - /* TABLE_1 - table container */ - - description "TABLE_1 description"; - - container OBJECT_1 { - /* OBJECT_1 - object container */ - - description "OBJECT_1 description"; - } - } - } -} diff --git a/tests/cli_autogen_input/yang_parser_test/sonic-1-table-container.yang b/tests/cli_autogen_input/yang_parser_test/sonic-1-table-container.yang deleted file mode 100644 index 36b98415e5..0000000000 --- a/tests/cli_autogen_input/yang_parser_test/sonic-1-table-container.yang +++ /dev/null @@ -1,17 +0,0 @@ -module sonic-1-table-container { - - yang-version 1.1; - - namespace "http://github.com/Azure/s-1-table"; - prefix s-1-table; - - container sonic-1-table-container { - /* sonic-1-table-container - top level container */ - - container TABLE_1 { - /* TABLE_1 - table container */ - - description "TABLE_1 description"; - } - } -} diff --git a/tests/cli_autogen_input/yang_parser_test/sonic-2-lists.yang b/tests/cli_autogen_input/yang_parser_test/sonic-2-lists.yang deleted file mode 100644 index fce9704f00..0000000000 --- a/tests/cli_autogen_input/yang_parser_test/sonic-2-lists.yang +++ /dev/null @@ -1,42 +0,0 @@ -module sonic-2-lists { - - yang-version 1.1; - - namespace "http://github.com/Azure/s-2-lists"; - prefix s-2-lists; - - container sonic-2-lists { - /* sonic-2-lists - top level container */ - - container TABLE_1 { - /* TALBE_1 - table container */ - - - description "TABLE_1 description"; - - list TABLE_1_LIST_1 { - /* TALBE_1_LIST_1 - object container */ - - description "TABLE_1_LIST_1 description"; - - key "key_name1"; - - leaf key_name1 { - type string; - } - } - - list TABLE_1_LIST_2 { - /* TALBE_1_LIST_2 - object container */ - - description "TABLE_1_LIST_2 description"; - - key "key_name2"; - - leaf key_name2 { - type string; - } - } - } - } -} diff --git a/tests/cli_autogen_input/yang_parser_test/sonic-2-object-containers.yang b/tests/cli_autogen_input/yang_parser_test/sonic-2-object-containers.yang deleted file mode 100644 index e633b66246..0000000000 --- a/tests/cli_autogen_input/yang_parser_test/sonic-2-object-containers.yang +++ /dev/null @@ -1,29 +0,0 @@ -module sonic-2-object-containers { - - yang-version 1.1; - - namespace "http://github.com/Azure/s-2-object"; - prefix s-2-object; - - container sonic-2-object-containers { - /* sonic-2-object-containers - top level container */ - - container TABLE_1 { - /* TABLE_1 - table container */ - - description "FIRST_TABLE description"; - - container OBJECT_1 { - /* OBJECT_1 - object container */ - - description "OBJECT_1 description"; - } - - container OBJECT_2 { - /* OBJECT_2 - object container */ - - description "OBJECT_2 description"; - } - } - } -} diff --git a/tests/cli_autogen_input/yang_parser_test/sonic-2-table-containers.yang b/tests/cli_autogen_input/yang_parser_test/sonic-2-table-containers.yang deleted file mode 100644 index f5284c67ee..0000000000 --- a/tests/cli_autogen_input/yang_parser_test/sonic-2-table-containers.yang +++ /dev/null @@ -1,23 +0,0 @@ -module sonic-2-table-containers { - - yang-version 1.1; - - namespace "http://github.com/Azure/s-2-table"; - prefix s-2-table; - - container sonic-2-table-containers { - /* sonic-2-table-containers - top level container */ - - container TABLE_1 { - /* TABLE_1 - table container */ - - description "TABLE_1 description"; - } - - container TABLE_2 { - /* TABLE_2 - table container */ - - description "TABLE_2 description"; - } - } -} diff --git a/tests/cli_autogen_input/yang_parser_test/sonic-choice-complex.yang b/tests/cli_autogen_input/yang_parser_test/sonic-choice-complex.yang deleted file mode 100644 index 9d6e0de9ee..0000000000 --- a/tests/cli_autogen_input/yang_parser_test/sonic-choice-complex.yang +++ /dev/null @@ -1,91 +0,0 @@ -module sonic-choice-complex { - - yang-version 1.1; - - namespace "http://github.com/Azure/choice-complex"; - prefix choice-complex; - - import sonic-grouping-1 { - prefix sgroup1; - } - - import sonic-grouping-2 { - prefix sgroup2; - } - - grouping GR_5 { - leaf GR_5_LEAF_1 { - type string; - } - - leaf GR_5_LEAF_2 { - type string; - } - } - - grouping GR_6 { - leaf GR_6_LEAF_1 { - type string; - } - - leaf GR_6_LEAF_2 { - type string; - } - } - - container sonic-choice-complex { - /* sonic-choice-complex - top level container */ - - container TABLE_1 { - /* TABLE_1 - table container */ - - description "TABLE_1 description"; - - container OBJECT_1 { - /* OBJECT_1 - object container, it have - * 1 choice, which have 2 cases. - * first case have: 1 leaf, 1 leaf-list, 1 uses - * second case have: 2 leafs, 2 leaf-lists, 2 uses - */ - - description "OBJECT_1 description"; - - choice CHOICE_1 { - case CHOICE_1_CASE_1 { - leaf LEAF_1 { - type uint16; - } - - leaf-list LEAF_LIST_1 { - type string; - } - - uses sgroup1:GR_1; - } - - case CHOICE_1_CASE_2 { - leaf LEAF_2 { - type string; - } - - leaf LEAF_3 { - type string; - } - - leaf-list LEAF_LIST_2 { - type string; - } - - leaf-list LEAF_LIST_3 { - type string; - } - - uses GR_5; - uses sgroup1:GR_2; - uses sgroup2:GR_3; - } - } - } - } - } -} diff --git a/tests/cli_autogen_input/yang_parser_test/sonic-dynamic-object-complex-1.yang b/tests/cli_autogen_input/yang_parser_test/sonic-dynamic-object-complex-1.yang deleted file mode 100644 index 383e94fb43..0000000000 --- a/tests/cli_autogen_input/yang_parser_test/sonic-dynamic-object-complex-1.yang +++ /dev/null @@ -1,57 +0,0 @@ -module sonic-dynamic-object-complex-1 { - - yang-version 1.1; - - namespace "http://github.com/Azure/dynamic-complex-1"; - prefix dynamic-complex-1; - - container sonic-dynamic-object-complex-1 { - /* sonic-dynamic-object-complex-1 - top level container */ - - container TABLE_1 { - /* TABLE_1 - table container */ - - description "TABLE_1 description"; - - list OBJECT_1_LIST { - /* OBJECT_1_LIST - dynamic object container, it have: - * 1 key, - * 1 leaf, - * 1 leaf-list - * 1 choice - */ - - description "OBJECT_1_LIST description"; - - key "KEY_LEAF_1"; - - leaf KEY_LEAF_1 { - description "KEY_LEAF_1 description"; - type string; - } - - leaf OBJ_1_LEAF_1 { - description "OBJ_1_LEAF_1 description"; - type string; - } - - leaf-list OBJ_1_LEAF_LIST_1 { - type string; - } - - choice OBJ_1_CHOICE_1 { - case OBJ_1_CHOICE_1_CASE_1 { - leaf OBJ_1_CHOICE_1_LEAF_1 { - type uint16; - } - } - case OBJ_1_CHOICE_1_CASE_2 { - leaf OBJ_1_CHOICE_1_LEAF_2 { - type string; - } - } - } - } - } - } -} diff --git a/tests/cli_autogen_input/yang_parser_test/sonic-dynamic-object-complex-2.yang b/tests/cli_autogen_input/yang_parser_test/sonic-dynamic-object-complex-2.yang deleted file mode 100644 index a365b014ad..0000000000 --- a/tests/cli_autogen_input/yang_parser_test/sonic-dynamic-object-complex-2.yang +++ /dev/null @@ -1,84 +0,0 @@ -module sonic-dynamic-object-complex-2 { - - yang-version 1.1; - - namespace "http://github.com/Azure/dynamic-complex-2"; - prefix dynamic-complex-2; - - container sonic-dynamic-object-complex-2 { - /* sonic-dynamic-object-complex-2 - top level container */ - - container TABLE_1 { - /* TABLE_1 - table container */ - - description "TABLE_1 description"; - - list OBJECT_1_LIST { - /* OBJECT_1_LIST - dynamic object container, it have: - * 2 keys - * 2 leaf, - * 2 leaf-list - * 2 choice - */ - - description "OBJECT_1_LIST description"; - - key "KEY_LEAF_1 KEY_LEAF_2"; - - leaf KEY_LEAF_1 { - description "KEY_LEAF_1 description"; - type string; - } - - leaf KEY_LEAF_2 { - description "KEY_LEAF_2 description"; - type string; - } - - leaf OBJ_1_LEAF_1 { - description "OBJ_1_LEAF_1 description"; - type string; - } - - leaf OBJ_1_LEAF_2 { - description "OBJ_1_LEAF_2 description"; - type string; - } - - leaf-list OBJ_1_LEAF_LIST_1 { - type string; - } - - leaf-list OBJ_1_LEAF_LIST_2 { - type string; - } - - choice OBJ_1_CHOICE_1 { - case OBJ_1_CHOICE_1_CASE_1 { - leaf OBJ_1_CHOICE_1_LEAF_1 { - type uint16; - } - } - case OBJ_1_CHOICE_1_CASE_2 { - leaf OBJ_1_CHOICE_1_LEAF_2 { - type string; - } - } - } - - choice OBJ_1_CHOICE_2 { - case OBJ_1_CHOICE_2_CASE_1 { - leaf OBJ_1_CHOICE_2_LEAF_1 { - type uint16; - } - } - case OBJ_1_CHOICE_2_CASE_2 { - leaf OBJ_1_CHOICE_2_LEAF_2 { - type string; - } - } - } - } - } - } -} diff --git a/tests/cli_autogen_input/yang_parser_test/sonic-grouping-1.yang b/tests/cli_autogen_input/yang_parser_test/sonic-grouping-1.yang deleted file mode 100644 index bf0be792f5..0000000000 --- a/tests/cli_autogen_input/yang_parser_test/sonic-grouping-1.yang +++ /dev/null @@ -1,25 +0,0 @@ -module sonic-grouping-1{ - - yang-version 1.1; - - namespace "http://github.com/Azure/s-grouping-1"; - prefix s-grouping-1; - - grouping GR_1 { - leaf GR_1_LEAF_1 { - type string; - } - leaf GR_1_LEAF_2 { - type string; - } - } - - grouping GR_2 { - leaf GR_2_LEAF_1 { - type string; - } - leaf GR_2_LEAF_2 { - type string; - } - } -} diff --git a/tests/cli_autogen_input/yang_parser_test/sonic-grouping-2.yang b/tests/cli_autogen_input/yang_parser_test/sonic-grouping-2.yang deleted file mode 100644 index 58e9df6621..0000000000 --- a/tests/cli_autogen_input/yang_parser_test/sonic-grouping-2.yang +++ /dev/null @@ -1,25 +0,0 @@ -module sonic-grouping-2 { - - yang-version 1.1; - - namespace "http://github.com/Azure/s-grouping-2"; - prefix s-grouping-2; - - grouping GR_3 { - leaf GR_3_LEAF_1 { - type string; - } - leaf GR_3_LEAF_2 { - type string; - } - } - - grouping GR_4 { - leaf GR_4_LEAF_1 { - type string; - } - leaf GR_4_LEAF_2 { - type string; - } - } -} diff --git a/tests/cli_autogen_input/yang_parser_test/sonic-grouping-complex.yang b/tests/cli_autogen_input/yang_parser_test/sonic-grouping-complex.yang deleted file mode 100644 index 22956789b0..0000000000 --- a/tests/cli_autogen_input/yang_parser_test/sonic-grouping-complex.yang +++ /dev/null @@ -1,96 +0,0 @@ -module sonic-grouping-complex { - - yang-version 1.1; - - namespace "http://github.com/Azure/grouping-complex"; - prefix grouping-complex; - - import sonic-grouping-1 { - prefix sgroup1; - } - - import sonic-grouping-2 { - prefix sgroup2; - } - - grouping GR_5 { - leaf GR_5_LEAF_1 { - type string; - } - - leaf-list GR_5_LEAF_LIST_1 { - type string; - } - } - - grouping GR_6 { - leaf GR_6_LEAF_1 { - type string; - } - - leaf GR_6_LEAF_2 { - type string; - } - - choice GR_6_CHOICE_1 { - case CHOICE_1_CASE_1 { - leaf GR_6_CASE_1_LEAF_1 { - type uint16; - } - - leaf-list GR_6_CASE_1_LEAF_LIST_1 { - type string; - } - } - - case CHOICE_1_CASE_2 { - leaf GR_6_CASE_2_LEAF_1 { - type uint16; - } - - leaf GR_6_CASE_2_LEAF_2 { - type uint16; - } - - leaf-list GR_6_CASE_2_LEAF_LIST_1 { - type string; - } - - leaf-list GR_6_CASE_2_LEAF_LIST_2 { - type string; - } - } - } - } - - container sonic-grouping-complex { - /* sonic-grouping-complex - top level container */ - - container TABLE_1 { - /* TABLE_1 - table container */ - - description "TABLE_1 description"; - - container OBJECT_1 { - /* OBJECT_1 - object container, it have - * 1 choice, which have 2 cases. - * first case have: 1 leaf, 1 leaf-list, 1 uses - * second case have: 2 leafs, 2 leaf-lists, 2 uses - */ - - description "OBJECT_1 description"; - - uses sgroup1:GR_1; - } - - container OBJECT_2 { - - description "OBJECT_2 description"; - - uses GR_5; - uses GR_6; - uses sgroup2:GR_4; - } - } - } -} diff --git a/tests/cli_autogen_input/yang_parser_test/sonic-static-object-complex-1.yang b/tests/cli_autogen_input/yang_parser_test/sonic-static-object-complex-1.yang deleted file mode 100644 index fa082d3b25..0000000000 --- a/tests/cli_autogen_input/yang_parser_test/sonic-static-object-complex-1.yang +++ /dev/null @@ -1,49 +0,0 @@ -module sonic-static-object-complex-1 { - - yang-version 1.1; - - namespace "http://github.com/Azure/static-complex-1"; - prefix static-complex-1; - - container sonic-static-object-complex-1 { - /* sonic-static-object-complex-1 - top level container */ - - container TABLE_1 { - /* TABLE_1 - table container */ - - description "TABLE_1 description"; - - container OBJECT_1 { - /* OBJECT_1 - object container, it have: - * 1 leaf, - * 1 leaf-list - * 1 choice - */ - - description "OBJECT_1 description"; - - leaf OBJ_1_LEAF_1 { - description "OBJ_1_LEAF_1 description"; - type string; - } - - leaf-list OBJ_1_LEAF_LIST_1 { - type string; - } - - choice OBJ_1_CHOICE_1 { - case OBJ_1_CHOICE_1_CASE_1 { - leaf OBJ_1_CHOICE_1_LEAF_1 { - type uint16; - } - } - case OBJ_1_CHOICE_1_CASE_2 { - leaf OBJ_1_CHOICE_1_LEAF_2 { - type string; - } - } - } - } - } - } -} diff --git a/tests/cli_autogen_input/yang_parser_test/sonic-static-object-complex-2.yang b/tests/cli_autogen_input/yang_parser_test/sonic-static-object-complex-2.yang deleted file mode 100644 index 4e53b2e1b1..0000000000 --- a/tests/cli_autogen_input/yang_parser_test/sonic-static-object-complex-2.yang +++ /dev/null @@ -1,71 +0,0 @@ -module sonic-static-object-complex-2 { - - yang-version 1.1; - - namespace "http://github.com/Azure/static-complex-2"; - prefix static-complex-2; - - container sonic-static-object-complex-2 { - /* sonic-static-object-complex-2 - top level container */ - - container TABLE_1 { - /* TABLE_1 - table container */ - - description "TABLE_1 description"; - - container OBJECT_1 { - /* OBJECT_1 - object container, it have: - * 2 leafs, - * 2 leaf-lists, - * 2 choices - */ - - description "OBJECT_1 description"; - - leaf OBJ_1_LEAF_1 { - description "OBJ_1_LEAF_1 description"; - type string; - } - - leaf OBJ_1_LEAF_2 { - description "OBJ_1_LEAF_2 description"; - type string; - } - - leaf-list OBJ_1_LEAF_LIST_1 { - type string; - } - - leaf-list OBJ_1_LEAF_LIST_2 { - type string; - } - - choice OBJ_1_CHOICE_1 { - case OBJ_1_CHOICE_1_CASE_1 { - leaf OBJ_1_CHOICE_1_LEAF_1 { - type uint16; - } - } - case OBJ_1_CHOICE_1_CASE_2 { - leaf OBJ_1_CHOICE_1_LEAF_2 { - type string; - } - } - } - - choice OBJ_1_CHOICE_2 { - case OBJ_1_CHOICE_2_CASE_1 { - leaf OBJ_1_CHOICE_2_LEAF_1 { - type uint16; - } - } - case OBJ_1_CHOICE_2_CASE_2 { - leaf OBJ_1_CHOICE_2_LEAF_2 { - type string; - } - } - } - } - } - } -} diff --git a/tests/cli_autogen_test.py b/tests/cli_autogen_test.py deleted file mode 100644 index 13407d1c13..0000000000 --- a/tests/cli_autogen_test.py +++ /dev/null @@ -1,243 +0,0 @@ -import os -import logging -import pytest - -import show.plugins as show_plugins -import show.main as show_main -import config.plugins as config_plugins -import config.main as config_main -from .cli_autogen_input.autogen_test import show_cmd_output -from .cli_autogen_input.cli_autogen_common import backup_yang_models, restore_backup_yang_models, move_yang_models, remove_yang_models - -from utilities_common import util_base -from sonic_cli_gen.generator import CliGenerator -from .mock_tables import dbconnector -from utilities_common.db import Db -from click.testing import CliRunner - -logger = logging.getLogger(__name__) -gen = CliGenerator(logger) - -test_path = os.path.dirname(os.path.abspath(__file__)) -mock_db_path = os.path.join(test_path, 'cli_autogen_input', 'config_db') -config_db_path = os.path.join(test_path, 'cli_autogen_input', 'config_db.json') -templates_path = os.path.join(test_path, '../', 'sonic-utilities-data', 'templates', 'sonic-cli-gen') - -SUCCESS = 0 -ERROR = 1 -INVALID_VALUE = 'INVALID' - -test_yang_models = [ - 'sonic-device_metadata.yang', - 'sonic-device_neighbor.yang', -] - - -class TestCliAutogen: - @classmethod - def setup_class(cls): - logger.info('SETUP') - os.environ['UTILITIES_UNIT_TESTING'] = '2' - - backup_yang_models() - move_yang_models(test_path, 'autogen_test', test_yang_models) - - for yang_model in test_yang_models: - gen.generate_cli_plugin( - cli_group='show', - plugin_name=yang_model.split('.')[0], - config_db_path=config_db_path, - templates_path=templates_path - ) - gen.generate_cli_plugin( - cli_group='config', - plugin_name=yang_model.split('.')[0], - config_db_path=config_db_path, - templates_path=templates_path - ) - - helper = util_base.UtilHelper() - helper.load_and_register_plugins(show_plugins, show_main.cli) - helper.load_and_register_plugins(config_plugins, config_main.config) - - - @classmethod - def teardown_class(cls): - logger.info('TEARDOWN') - - for yang_model in test_yang_models: - gen.remove_cli_plugin('show', yang_model.split('.')[0]) - gen.remove_cli_plugin('config', yang_model.split('.')[0]) - - restore_backup_yang_models() - - dbconnector.dedicated_dbs['CONFIG_DB'] = None - - os.environ['UTILITIES_UNIT_TESTING'] = '0' - - - def test_show_device_metadata(self): - dbconnector.dedicated_dbs['CONFIG_DB'] = mock_db_path - db = Db() - runner = CliRunner() - - result = runner.invoke( - show_main.cli.commands['device-metadata'].commands['localhost'], [], obj=db - ) - - logger.debug("\n" + result.output) - logger.debug(result.exit_code) - assert result.exit_code == SUCCESS - assert result.output == show_cmd_output.show_device_metadata_localhost - - - def test_config_device_metadata(self): - dbconnector.dedicated_dbs['CONFIG_DB'] = mock_db_path - db = Db() - runner = CliRunner() - - result = runner.invoke( - config_main.config.commands['device-metadata'].commands['localhost'].commands['buffer-model'], ['dynamic'], obj=db - ) - - result = runner.invoke( - show_main.cli.commands['device-metadata'].commands['localhost'], [], obj=db - ) - - logger.debug("\n" + result.output) - logger.debug(result.exit_code) - assert result.exit_code == SUCCESS - assert result.output == show_cmd_output.show_device_metadata_localhost_changed_buffer_model - - - @pytest.mark.parametrize("parameter,value", [ - ('default-bgp-status', INVALID_VALUE), - ('docker-routing-config-mode', INVALID_VALUE), - ('mac', INVALID_VALUE), - ('default-pfcwd-status', INVALID_VALUE), - ('bgp-asn', INVALID_VALUE), - ('type', INVALID_VALUE), - ('buffer-model', INVALID_VALUE), - ('frr-mgmt-framework-config', INVALID_VALUE) - ]) - def test_config_device_metadata_invalid(self, parameter, value): - dbconnector.dedicated_dbs['CONFIG_DB'] = mock_db_path - db = Db() - runner = CliRunner() - - result = runner.invoke( - config_main.config.commands['device-metadata'].commands['localhost'].commands[parameter], [value], obj=db - ) - - logger.debug("\n" + result.output) - logger.debug(result.exit_code) - assert result.exit_code == ERROR - - - def test_show_device_neighbor(self): - dbconnector.dedicated_dbs['CONFIG_DB'] = mock_db_path - db = Db() - runner = CliRunner() - - result = runner.invoke( - show_main.cli.commands['device-neighbor'], [], obj=db - ) - - logger.debug("\n" + result.output) - logger.debug(result.exit_code) - assert show_cmd_output.show_device_neighbor - assert result.exit_code == SUCCESS - - - def test_config_device_neighbor_add(self): - dbconnector.dedicated_dbs['CONFIG_DB'] = mock_db_path - db = Db() - runner = CliRunner() - - result = runner.invoke( - config_main.config.commands['device-neighbor'].commands['add'], - ['Ethernet8', '--name', 'Servers1', '--mgmt-addr', '10.217.0.3', - '--local-port', 'Ethernet8', '--port', 'eth2', '--type', 'type'], - obj=db - ) - logger.debug("\n" + result.output) - logger.debug(result.exit_code) - - result = runner.invoke( - show_main.cli.commands['device-neighbor'], [], obj=db - ) - - logger.debug("\n" + result.output) - logger.debug(result.exit_code) - assert result.exit_code == SUCCESS - assert result.output == show_cmd_output.show_device_neighbor_added - - - def test_config_device_neighbor_delete(self): - dbconnector.dedicated_dbs['CONFIG_DB'] = mock_db_path - db = Db() - runner = CliRunner() - - result = runner.invoke( - config_main.config.commands['device-neighbor'].commands['delete'], - ['Ethernet0'], obj=db - ) - logger.debug("\n" + result.output) - logger.debug(result.exit_code) - - result = runner.invoke( - show_main.cli.commands['device-neighbor'], [], obj=db - ) - - logger.debug("\n" + result.output) - logger.debug(result.exit_code) - assert result.exit_code == SUCCESS - assert result.output == show_cmd_output.show_device_neighbor_deleted - - - @pytest.mark.parametrize("parameter,value,output", [ - ('--mgmt-addr', '10.217.0.5', show_cmd_output.show_device_neighbor_updated_mgmt_addr), - ('--name', 'Servers1', show_cmd_output.show_device_neighbor_updated_name), - ('--local-port', 'Ethernet12', show_cmd_output.show_device_neighbor_updated_local_port), - ('--port', 'eth2', show_cmd_output.show_device_neighbor_updated_port), - ('--type', 'type2', show_cmd_output.show_device_neighbor_updated_type), - ]) - def test_config_device_neighbor_update(self, parameter, value, output): - dbconnector.dedicated_dbs['CONFIG_DB'] = mock_db_path - db = Db() - runner = CliRunner() - - result = runner.invoke( - config_main.config.commands['device-neighbor'].commands['update'], - ['Ethernet0', parameter, value], obj=db - ) - logger.debug("\n" + result.output) - logger.debug(result.exit_code) - - result = runner.invoke( - show_main.cli.commands['device-neighbor'], [], obj=db - ) - - logger.debug("\n" + result.output) - logger.debug(result.exit_code) - assert result.exit_code == SUCCESS - assert result.output == output - - - @pytest.mark.parametrize("parameter,value", [ - ('--mgmt-addr', INVALID_VALUE), - ('--local-port', INVALID_VALUE) - ]) - def test_config_device_neighbor_update_invalid(self, parameter, value): - dbconnector.dedicated_dbs['CONFIG_DB'] = mock_db_path - db = Db() - runner = CliRunner() - - result = runner.invoke( - config_main.config.commands['device-neighbor'].commands['update'], - ['Ethernet0', parameter, value], obj=db - ) - logger.debug("\n" + result.output) - logger.debug(result.exit_code) - assert result.exit_code == ERROR - diff --git a/tests/cli_autogen_yang_parser_test.py b/tests/cli_autogen_yang_parser_test.py deleted file mode 100644 index ed82693e91..0000000000 --- a/tests/cli_autogen_yang_parser_test.py +++ /dev/null @@ -1,172 +0,0 @@ -import os -import logging -import pprint - -from sonic_cli_gen.yang_parser import YangParser -from .cli_autogen_input.yang_parser_test import assert_dictionaries -from .cli_autogen_input.cli_autogen_common import move_yang_models, remove_yang_models - -logger = logging.getLogger(__name__) - -test_path = os.path.dirname(os.path.abspath(__file__)) - -test_yang_models = [ - 'sonic-1-table-container.yang', - 'sonic-2-table-containers.yang', - 'sonic-1-object-container.yang', - 'sonic-2-object-containers.yang', - 'sonic-1-list.yang', - 'sonic-2-lists.yang', - 'sonic-static-object-complex-1.yang', - 'sonic-static-object-complex-2.yang', - 'sonic-dynamic-object-complex-1.yang', - 'sonic-dynamic-object-complex-2.yang', - 'sonic-choice-complex.yang', - 'sonic-grouping-complex.yang', - 'sonic-grouping-1.yang', - 'sonic-grouping-2.yang', -] - - -class TestYangParser: - @classmethod - def setup_class(cls): - logger.info("SETUP") - os.environ['UTILITIES_UNIT_TESTING'] = "2" - move_yang_models(test_path, 'yang_parser_test', test_yang_models) - - @classmethod - def teardown_class(cls): - logger.info("TEARDOWN") - os.environ['UTILITIES_UNIT_TESTING'] = "0" - remove_yang_models(test_yang_models) - - def test_1_table_container(self): - """ Test for 1 'table' container - 'table' container represent TABLE in Config DB schema: - { - "TABLE": { - "OBJECT": { - "attr": "value" - ... - } - } - } - """ - - base_test('sonic-1-table-container', - assert_dictionaries.one_table_container) - - def test_2_table_containers(self): - """ Test for 2 'table' containers """ - - base_test('sonic-2-table-containers', - assert_dictionaries.two_table_containers) - - def test_1_object_container(self): - """ Test for 1 'object' container - 'object' container represent OBJECT in Config DB schema: - { - "TABLE": { - "OBJECT": { - "attr": "value" - ... - } - } - } - """ - - base_test('sonic-1-object-container', - assert_dictionaries.one_object_container) - - def test_2_object_containers(self): - """ Test for 2 'object' containers """ - - base_test('sonic-2-object-containers', - assert_dictionaries.two_object_containers) - - def test_1_list(self): - """ Test for 1 container that has inside - the YANG 'list' entity - """ - - base_test('sonic-1-list', assert_dictionaries.one_list) - - def test_2_lists(self): - """ Test for 2 containers that have inside - the YANG 'list' entity - """ - - base_test('sonic-2-lists', assert_dictionaries.two_lists) - - def test_static_object_complex_1(self): - """ Test for the object container with: - 1 leaf, 1 leaf-list, 1 choice. - """ - - base_test('sonic-static-object-complex-1', - assert_dictionaries.static_object_complex_1) - - def test_static_object_complex_2(self): - """ Test for object container with: - 2 leafs, 2 leaf-lists, 2 choices. - """ - - base_test('sonic-static-object-complex-2', - assert_dictionaries.static_object_complex_2) - - def test_dynamic_object_complex_1(self): - """ Test for object container with: - 1 key, 1 leaf, 1 leaf-list, 1 choice. - """ - - base_test('sonic-dynamic-object-complex-1', - assert_dictionaries.dynamic_object_complex_1) - - def test_dynamic_object_complex_2(self): - """ Test for object container with: - 2 keys, 2 leafs, 2 leaf-list, 2 choice. - """ - - base_test('sonic-dynamic-object-complex-2', - assert_dictionaries.dynamic_object_complex_2) - - def test_choice_complex(self): - """ Test for object container with the 'choice' - that have complex strucutre: - leafs, leaf-lists, multiple 'uses' from different files - """ - - base_test('sonic-choice-complex', - assert_dictionaries.choice_complex) - - def test_grouping_complex(self): - """ Test for object container with multitple 'uses' that using 'grouping' - from different files. The used 'grouping' have a complex structure: - leafs, leaf-lists, choices - """ - - base_test('sonic-grouping-complex', - assert_dictionaries.grouping_complex) - - -def base_test(yang_model_name, correct_dict): - """ General logic for each test case """ - - config_db_path = os.path.join(test_path, - 'mock_tables/config_db.json') - parser = YangParser(yang_model_name=yang_model_name, - config_db_path=config_db_path, - allow_tbl_without_yang=True, - debug=False) - yang_dict = parser.parse_yang_model() - pretty_log_debug(yang_dict) - assert yang_dict == correct_dict - - -def pretty_log_debug(dictionary): - """ Pretty print of parsed dictionary """ - - for line in pprint.pformat(dictionary).split('\n'): - logging.debug(line) - diff --git a/utilities_common/util_base.py b/utilities_common/util_base.py deleted file mode 100644 index 98fc230629..0000000000 --- a/utilities_common/util_base.py +++ /dev/null @@ -1,91 +0,0 @@ -import os -import pkgutil -import importlib - -from sonic_py_common import logger - -# Constants ==================================================================== -PDDF_SUPPORT_FILE = '/usr/share/sonic/platform/pddf_support' - -# Helper classs - -log = logger.Logger() - - -class UtilHelper(object): - def __init__(self): - pass - - def load_plugins(self, plugins_namespace): - """ Discover and load CLI plugins. Yield a plugin module. """ - - def iter_namespace(ns_pkg): - return pkgutil.iter_modules(ns_pkg.__path__, ns_pkg.__name__ + ".") - - for _, module_name, ispkg in iter_namespace(plugins_namespace): - if ispkg: - yield from self.load_plugins(importlib.import_module(module_name)) - continue - log.log_debug('importing plugin: {}'.format(module_name)) - try: - module = importlib.import_module(module_name) - except Exception as err: - log.log_error('failed to import plugin {}: {}'.format(module_name, err), - also_print_to_console=True) - continue - - yield module - - def register_plugin(self, plugin, root_command): - """ Register plugin in top-level command root_command. """ - - name = plugin.__name__ - log.log_debug('registering plugin: {}'.format(name)) - try: - plugin.register(root_command) - except Exception as err: - log.log_error('failed to import plugin {}: {}'.format(name, err), - also_print_to_console=True) - - # try get information from platform API and return a default value if caught NotImplementedError - def try_get(self, callback, default=None): - """ - Handy function to invoke the callback, catch NotImplementedError and return a default value - :param callback: Callback to be invoked - :param default: Default return value if exception occur - :return: Default return value if exception occur else return value of the callback - """ - try: - ret = callback() - if ret is None: - ret = default - except NotImplementedError: - ret = default - - return ret - - # Instantiate platform-specific Chassis class - def load_platform_chassis(self): - chassis = None - - # Load 2.0 platform API chassis class - try: - import sonic_platform - chassis = sonic_platform.platform.Platform().get_chassis() - except Exception as e: - raise Exception("Failed to load chassis due to {}".format(repr(e))) - - return chassis - - # Check for PDDF mode enabled - def check_pddf_mode(self): - if os.path.exists(PDDF_SUPPORT_FILE): - return True - else: - return False - - def load_and_register_plugins(self, plugins, cli): - """ Load plugins and register them """ - - for plugin in self.load_plugins(plugins): - self.register_plugin(plugin, cli) \ No newline at end of file From 521990975b23315338e6d36216d3396df5223bd3 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Wed, 10 Nov 2021 20:20:03 +0200 Subject: [PATCH 170/173] remove sonic_cli_gen PR part Signed-off-by: Stepan Blyschak --- clear/main.py | 1 + config/main.py | 2 +- setup.py | 5 --- utilities_common/util_base.py | 85 +++++++++++++++++++++++++++++++++++ 4 files changed, 87 insertions(+), 6 deletions(-) create mode 100644 utilities_common/util_base.py diff --git a/clear/main.py b/clear/main.py index 44fa75c726..5cbaee9d62 100755 --- a/clear/main.py +++ b/clear/main.py @@ -487,5 +487,6 @@ def remap_keys(dict): for plugin in helper.load_plugins(plugins): helper.register_plugin(plugin, cli) + if __name__ == '__main__': cli() diff --git a/config/main.py b/config/main.py index e7f03cb4e2..1b2678a4ef 100644 --- a/config/main.py +++ b/config/main.py @@ -5980,7 +5980,7 @@ def smoothing_interval(interval, rates_type): # Load plugins and register them helper = util_base.UtilHelper() for plugin in helper.load_plugins(plugins): - helper.register_plugin(plugin, cli) + helper.register_plugin(plugin, config) if __name__ == '__main__': diff --git a/setup.py b/setup.py index f9979a8f4f..30e8c7bfd4 100644 --- a/setup.py +++ b/setup.py @@ -23,10 +23,8 @@ 'acl_loader', 'clear', 'clear.plugins', - 'clear.plugins.auto', 'config', 'config.plugins', - 'config.plugins.auto', 'connect', 'consutil', 'counterpoll', @@ -50,7 +48,6 @@ 'show', 'show.interfaces', 'show.plugins', - 'show.plugins.auto', 'sonic_installer', 'sonic_installer.bootloader', 'sonic_package_manager', @@ -59,7 +56,6 @@ 'undebug', 'utilities_common', 'watchdogutil', - 'sonic_cli_gen', ], package_data={ 'show': ['aliases.ini'], @@ -173,7 +169,6 @@ 'spm = sonic_package_manager.main:cli', 'undebug = undebug.main:cli', 'watchdogutil = watchdogutil.main:watchdogutil', - 'sonic-cli-gen = sonic_cli_gen.main:cli', ] }, install_requires=[ diff --git a/utilities_common/util_base.py b/utilities_common/util_base.py new file mode 100644 index 0000000000..f7b5188e15 --- /dev/null +++ b/utilities_common/util_base.py @@ -0,0 +1,85 @@ +import os +import pkgutil +import importlib + +from sonic_py_common import logger + +# Constants ==================================================================== +PDDF_SUPPORT_FILE = '/usr/share/sonic/platform/pddf_support' + +# Helper classs + +log = logger.Logger() + + +class UtilHelper(object): + def __init__(self): + pass + + def load_plugins(self, plugins_namespace): + """ Discover and load CLI plugins. Yield a plugin module. """ + + def iter_namespace(ns_pkg): + return pkgutil.iter_modules(ns_pkg.__path__, ns_pkg.__name__ + ".") + + for _, module_name, ispkg in iter_namespace(plugins_namespace): + if ispkg: + continue + log.log_debug('importing plugin: {}'.format(module_name)) + try: + module = importlib.import_module(module_name) + except Exception as err: + log.log_error('failed to import plugin {}: {}'.format(module_name, err), + also_print_to_console=True) + continue + + yield module + + def register_plugin(self, plugin, root_command): + """ Register plugin in top-level command root_command. """ + + name = plugin.__name__ + log.log_debug('registering plugin: {}'.format(name)) + try: + plugin.register(root_command) + except Exception as err: + log.log_error('failed to import plugin {}: {}'.format(name, err), + also_print_to_console=True) + + # try get information from platform API and return a default value if caught NotImplementedError + def try_get(self, callback, default=None): + """ + Handy function to invoke the callback, catch NotImplementedError and return a default value + :param callback: Callback to be invoked + :param default: Default return value if exception occur + :return: Default return value if exception occur else return value of the callback + """ + try: + ret = callback() + if ret is None: + ret = default + except NotImplementedError: + ret = default + + return ret + + # Instantiate platform-specific Chassis class + def load_platform_chassis(self): + chassis = None + + # Load 2.0 platform API chassis class + try: + import sonic_platform + chassis = sonic_platform.platform.Platform().get_chassis() + except Exception as e: + raise Exception("Failed to load chassis due to {}".format(repr(e))) + + return chassis + + # Check for PDDF mode enabled + def check_pddf_mode(self): + if os.path.exists(PDDF_SUPPORT_FILE): + return True + else: + return False + From 987c085ab7909d8b503246ff1e45e66e945ddcda Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Wed, 10 Nov 2021 20:21:32 +0200 Subject: [PATCH 171/173] remove sonic_cli_gen PR part Signed-off-by: Stepan Blyschak --- utilities_common/util_base.py | 1 - 1 file changed, 1 deletion(-) diff --git a/utilities_common/util_base.py b/utilities_common/util_base.py index f7b5188e15..ff5570735c 100644 --- a/utilities_common/util_base.py +++ b/utilities_common/util_base.py @@ -82,4 +82,3 @@ def check_pddf_mode(self): return True else: return False - From bf05d422c61a657a16800f581ad0ffb0b3d842b4 Mon Sep 17 00:00:00 2001 From: Stepan Blyshchak Date: Wed, 17 Nov 2021 03:31:42 +0200 Subject: [PATCH 172/173] handle review comments Signed-off-by: Stepan Blyshchak --- config/config_mgmt.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/config/config_mgmt.py b/config/config_mgmt.py index 4e34a7ae00..6c25560484 100644 --- a/config/config_mgmt.py +++ b/config/config_mgmt.py @@ -5,7 +5,9 @@ import os import re +import shutil import syslog +import tempfile import yang as ly from json import load from sys import flags @@ -220,7 +222,7 @@ def writeConfigDB(self, jDiff): return - def add_module(self, yang_module_str, replace_if_exists=False): + def add_module(self, yang_module_str): """ Validate and add new YANG module to the system. @@ -233,7 +235,7 @@ def add_module(self, yang_module_str, replace_if_exists=False): module_name = self.get_module_name(yang_module_str) module_path = os.path.join(YANG_DIR, '{}.yang'.format(module_name)) - if os.path.exists(module_path) and not replace_if_exists: + if os.path.exists(module_path): raise Exception('{} already exists'.format(module_name)) with open(module_path, 'w') as module_file: module_file.write(yang_module_str) @@ -245,7 +247,7 @@ def add_module(self, yang_module_str, replace_if_exists=False): def remove_module(self, module_name): """ - Remove YANG module on the system and validate. + Remove YANG module from the system and validate. Parameters: module_name (str): YANG module name. @@ -257,13 +259,12 @@ def remove_module(self, module_name): module_path = os.path.join(YANG_DIR, '{}.yang'.format(module_name)) if not os.path.exists(module_path): return - with open(module_path, 'r') as module_file: - yang_module_str = module_file.read() + temp_module_path = tempfile.mktemp() try: - os.remove(module_path) + shutil.move(module_path, temp_module_path) self.__init_sonic_yang() except Exception: - self.add_module(yang_module_str) + shutil.move(temp_module_path, module_path) raise @staticmethod From d0a2ff7bfb9651d57b46cc8aa9c3b653c74b7903 Mon Sep 17 00:00:00 2001 From: Stepan Blyshchak Date: Wed, 17 Nov 2021 05:22:41 +0200 Subject: [PATCH 173/173] fix LGTM warning Signed-off-by: Stepan Blyshchak --- config/config_mgmt.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/config/config_mgmt.py b/config/config_mgmt.py index 6c25560484..bbed677fa6 100644 --- a/config/config_mgmt.py +++ b/config/config_mgmt.py @@ -259,12 +259,12 @@ def remove_module(self, module_name): module_path = os.path.join(YANG_DIR, '{}.yang'.format(module_name)) if not os.path.exists(module_path): return - temp_module_path = tempfile.mktemp() + temp = tempfile.NamedTemporaryFile(delete=False) try: - shutil.move(module_path, temp_module_path) + shutil.move(module_path, temp.name) self.__init_sonic_yang() except Exception: - shutil.move(temp_module_path, module_path) + shutil.move(temp.name, module_path) raise @staticmethod