From f20073950fb7663dc6f91fc45c3b6ca81e96e7e6 Mon Sep 17 00:00:00 2001 From: Gaurav Mishra Date: Thu, 27 Jan 2022 16:32:53 +0530 Subject: [PATCH] feat(cli): Change app name to snap2bucket 1. Change the app name to snap2bucket. 2. Write Actions to build PyPi packages. 3. Update version to 1.0.4 Signed-off-by: Gaurav Mishra --- .github/workflows/docs.yml | 2 +- .github/workflows/lint.yml | 11 ++++-- .github/workflows/release-publish.yml | 37 +++++++++++++++++++ .gitignore | 1 + Pipfile | 4 ++ README.md | 22 +++++++---- doc-sources/conf.py | 4 +- doc-sources/index.rst | 14 +++++-- doc-sources/recovery.rst | 4 +- doc-sources/setupmigrate.rst | 4 +- setup.py | 6 +-- src/snap_to_bucket/__init__.py | 5 --- src/snap_to_bucket/handlers/__init__.py | 13 +++++++ .../{ => handlers}/ec_2_handler.py | 0 .../{ => handlers}/fs_handler.py | 0 .../{ => handlers}/s3_handler.py | 2 +- src/snap_to_bucket/run.py | 4 +- src/snap_to_bucket/runner/__init__.py | 11 ++++++ .../{ => runner}/snap_handler.py | 6 +-- 19 files changed, 115 insertions(+), 35 deletions(-) create mode 100644 .github/workflows/release-publish.yml create mode 100644 src/snap_to_bucket/handlers/__init__.py rename src/snap_to_bucket/{ => handlers}/ec_2_handler.py (100%) rename src/snap_to_bucket/{ => handlers}/fs_handler.py (100%) rename src/snap_to_bucket/{ => handlers}/s3_handler.py (99%) create mode 100644 src/snap_to_bucket/runner/__init__.py rename src/snap_to_bucket/{ => runner}/snap_handler.py (98%) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 750d4b6..f25421f 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -11,7 +11,7 @@ jobs: image: python:3.8-slim steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v3 - name: Install host dependencies run: | apt-get -qq update diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 50d0070..e3c9cb9 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -10,18 +10,21 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python 3.9 - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: 3.9 - name: Install dependencies run: | - python -m pip install --upgrade pip - pip install pylint pipenv + pipx install pylint + pipx install pipenv - name: Install package run: | pipenv install --dev --editable . - name: Analysing the code with pylint run: | pipenv run pylint src/snap_to_bucket + - name: Check package + run: | + pipenv run check_package diff --git a/.github/workflows/release-publish.yml b/.github/workflows/release-publish.yml new file mode 100644 index 0000000..e6ab301 --- /dev/null +++ b/.github/workflows/release-publish.yml @@ -0,0 +1,37 @@ +name: Publish Release Packages + +on: + release: + types: + - published + +jobs: + release: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + + - name: Setup python + uses: actions/setup-python@v4 + with: + python-version: '3.8' + architecture: 'x64' + + - name: Install build dependencies + run: | + pipx install pipenv + pipenv install + + - name: Check package + run: pipenv run check_package + + - name: Build packages + run: pipenv run build_packages + + - name: Upload Packages to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + user: __token__ + password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/.gitignore b/.gitignore index 7ffa447..20a1341 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ tags *__pycache__/ build *.egg-info +dist diff --git a/Pipfile b/Pipfile index cc6144e..c8e4a9b 100644 --- a/Pipfile +++ b/Pipfile @@ -13,3 +13,7 @@ snap-to-bucket = {path = "."} sphinx = "*" snap-to-bucket = {editable = true, path = "."} pylint = "*" + +[scripts] +build_packages = "python3 ./setup.py sdist bdist_wheel" +check_package = "python3 ./setup.py check" diff --git a/README.md b/README.md index 9b4320a..0e0c268 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,25 @@ # snap_to_bucket ![snap_to_bucket docs](https://github.com/siemens/snap-to-bucket/workflows/snap_to_bucket%20docs/badge.svg) +[![PyPI version](https://badge.fury.io/py/snap-to-bucket.svg)](https://badge.fury.io/py/snap-to-bucket) This tool allows to move data from AWS snapshots to S3 buckets. ### Installation +#### Local installation ```console +$ git clone https://github.com/siemens/snap-to-bucket.git +$ cd snap-to-bucket $ python3 -m pip install -U pipenv $ pipenv install --dev --editable . ``` +#### PyPi +```console +$ python3 -m pip install -U snap-to-bucket +``` + ### Requirements 1. The script needs to be running on an EC2 instance. @@ -95,11 +104,11 @@ script needs to mount/unmount volumes. - Runing from source ```console -# pipenv run snap_to_bucket --bucket +# pipenv run snap2bucket --bucket ``` - Runing from install ```console -# snap_to_bucket --bucket +# snap2bucket --bucket ``` If you have used different tags on snapshots, use `-t\--tag` option. @@ -133,10 +142,9 @@ Since the S3 has a limit on object size, a single split can not be larger than ### Options ``` -Usage: snap_to_bucket [OPTIONS] +Usage: snap2bucket [OPTIONS] - snap_to_bucket is a simple tool based on boto3 to move snapshots to S3 - buckets. + snap2bucket is a simple tool based on boto3 to move snapshots to S3 buckets. Options: --version Show the version and exit. @@ -239,11 +247,11 @@ for i in /sys /proc /run /dev; do sudo umount /mnt/snapshot$i; done Run the script with `-r\--restore` flag and provide the bucket and the key. - Runing from source ```console -# pipenv run snap_to_bucket --restore --bucket --key +# pipenv run snap2bucket --restore --bucket --key ``` - Runing from install ```console -# snap_to_bucket --restore --bucket --key +# snap2bucket --restore --bucket --key ``` **Note:** The script will create new volume of size 25% more than the size of diff --git a/doc-sources/conf.py b/doc-sources/conf.py index c144e16..f054485 100644 --- a/doc-sources/conf.py +++ b/doc-sources/conf.py @@ -25,11 +25,11 @@ # -- Project information ----------------------------------------------------- project = 'snap_to_bucket' -copyright = '2020-2021, Siemens AG' +copyright = '2020-2022, Siemens AG' author = 'Gaurav Mishra ' # The full version, including alpha/beta/rc tags -release = '1.0.1' +release = '1.0.4' # -- General configuration --------------------------------------------------- diff --git a/doc-sources/index.rst b/doc-sources/index.rst index 9c740a5..4b67479 100644 --- a/doc-sources/index.rst +++ b/doc-sources/index.rst @@ -11,11 +11,20 @@ This tool allows to move data from AWS snapshots to S3 buckets. Installation ======================= +* From source code + .. code-block:: bash + $ git clone https://github.com/siemens/snap-to-bucket.git + $ cd snap-to-bucket $ python3 -m pip install -U pipenv $ pipenv install --dev --editable . +* From PyPi + +.. code-block:: bash + $ python3 -m pip install -U snap-to-bucket + Requirements =============== @@ -95,10 +104,9 @@ Options .. code-block:: - Usage: snap_to_bucket [OPTIONS] + Usage: snap2bucket [OPTIONS] - snap_to_bucket is a simple tool based on boto3 to move snapshots to S3 - buckets. + snap2bucket is a simple tool based on boto3 to move snapshots to S3 buckets. Options: --version Show the version and exit. diff --git a/doc-sources/recovery.rst b/doc-sources/recovery.rst index dda50ed..c4b3c04 100644 --- a/doc-sources/recovery.rst +++ b/doc-sources/recovery.rst @@ -60,13 +60,13 @@ Run the script with ``-r\--restore`` flag and provide the bucket and the key. .. code-block:: bash - pipenv run snap_to_bucket --restore --bucket --key + pipenv run snap2bucket --restore --bucket --key * Runing from install .. code-block:: bash - snap_to_bucket --restore --bucket --key + snap2bucket --restore --bucket --key **Note:** The script will create new volume of size 25% more than the size of tar or ``x-amz-meta-disc-size`` metadata (if available). diff --git a/doc-sources/setupmigrate.rst b/doc-sources/setupmigrate.rst index c95278e..3f93aed 100644 --- a/doc-sources/setupmigrate.rst +++ b/doc-sources/setupmigrate.rst @@ -26,13 +26,13 @@ script needs to mount/unmount volumes. .. code-block:: bash - pipenv run snap_to_bucket --bucket + pipenv run snap2bucket --bucket * Runing from install .. code-block:: bash - snap_to_bucket --bucket + snap2bucket --bucket If you have used different tags on snapshots, use ``-t/--tag`` option. diff --git a/setup.py b/setup.py index 6dbe1c0..3de7591 100755 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- """ -SPDX-FileCopyrightText: Siemens AG, 2020-2021 Gaurav Mishra +SPDX-FileCopyrightText: Siemens AG, 2020-2022 Gaurav Mishra SPDX-License-Identifier: MIT """ @@ -32,7 +32,7 @@ def read(fname): metadata = dict( name="snap_to_bucket", - version="1.0.2", + version="1.0.4", author="Gaurav Mishra", author_email="mishra.gaurav@siemens.com", description=("Move AWS EBS Snapshots to S3 Buckets"), @@ -65,7 +65,7 @@ def read(fname): ], entry_points = { 'console_scripts': [ - 'snap_to_bucket = snap_to_bucket.run:main' + 'snap2bucket = snap_to_bucket.run:main' ] }, ) diff --git a/src/snap_to_bucket/__init__.py b/src/snap_to_bucket/__init__.py index d319632..fe7dad7 100644 --- a/src/snap_to_bucket/__init__.py +++ b/src/snap_to_bucket/__init__.py @@ -7,8 +7,3 @@ SPDX-License-Identifier: MIT """ __author__ = 'Siemens AG' - -from .ec_2_handler import Ec2Handler -from .fs_handler import FsHandler -from .s3_handler import S3Handler -from .snap_handler import SnapToBucket diff --git a/src/snap_to_bucket/handlers/__init__.py b/src/snap_to_bucket/handlers/__init__.py new file mode 100644 index 0000000..cadabec --- /dev/null +++ b/src/snap_to_bucket/handlers/__init__.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +""" +SPDX-FileCopyrightText: Siemens AG, 2022 Gaurav Mishra + +SPDX-License-Identifier: MIT +""" +__author__ = 'Siemens AG' + +from .ec_2_handler import Ec2Handler +from .s3_handler import S3Handler +from .fs_handler import FsHandler diff --git a/src/snap_to_bucket/ec_2_handler.py b/src/snap_to_bucket/handlers/ec_2_handler.py similarity index 100% rename from src/snap_to_bucket/ec_2_handler.py rename to src/snap_to_bucket/handlers/ec_2_handler.py diff --git a/src/snap_to_bucket/fs_handler.py b/src/snap_to_bucket/handlers/fs_handler.py similarity index 100% rename from src/snap_to_bucket/fs_handler.py rename to src/snap_to_bucket/handlers/fs_handler.py diff --git a/src/snap_to_bucket/s3_handler.py b/src/snap_to_bucket/handlers/s3_handler.py similarity index 99% rename from src/snap_to_bucket/s3_handler.py rename to src/snap_to_bucket/handlers/s3_handler.py index ac4d262..4d3fd7c 100644 --- a/src/snap_to_bucket/s3_handler.py +++ b/src/snap_to_bucket/handlers/s3_handler.py @@ -137,7 +137,7 @@ def __get_object_count(self, key): partition_size = int( response["Metadata"]["x-amz-meta-disc-size"]) if partition_size < 2: - partition_size = sum([int(o["Size"]) for o in objects]) + partition_size = sum((int(o["Size"]) for o in objects)) self.restore_partition_size = partition_size return len(objects) except Exception as ex: diff --git a/src/snap_to_bucket/run.py b/src/snap_to_bucket/run.py index 9732e2b..0e8bc9e 100755 --- a/src/snap_to_bucket/run.py +++ b/src/snap_to_bucket/run.py @@ -16,7 +16,7 @@ import click from pkg_resources import get_distribution -from snap_to_bucket import SnapToBucket +from snap_to_bucket.runner import SnapToBucket class VolSize(click.ParamType): @@ -122,7 +122,7 @@ def main(verbose, proxy, noproxy, bucket, tag, type, storage_class, mount, delete, split, gzip, restore, key, boot, restore_dir, iops, throughput): """ - snap_to_bucket is a simple tool based on boto3 to move snapshots to S3 + snap2bucket is a simple tool based on boto3 to move snapshots to S3 buckets. """ if type not in ["gp3", "io1", "io2"] and iops is not None: diff --git a/src/snap_to_bucket/runner/__init__.py b/src/snap_to_bucket/runner/__init__.py new file mode 100644 index 0000000..008f729 --- /dev/null +++ b/src/snap_to_bucket/runner/__init__.py @@ -0,0 +1,11 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +""" +SPDX-FileCopyrightText: Siemens AG, 2022 Gaurav Mishra + +SPDX-License-Identifier: MIT +""" +__author__ = 'Siemens AG' + +from .snap_handler import SnapToBucket diff --git a/src/snap_to_bucket/snap_handler.py b/src/snap_to_bucket/runner/snap_handler.py similarity index 98% rename from src/snap_to_bucket/snap_handler.py rename to src/snap_to_bucket/runner/snap_handler.py index b2214a3..11b5df6 100644 --- a/src/snap_to_bucket/snap_handler.py +++ b/src/snap_to_bucket/runner/snap_handler.py @@ -11,9 +11,9 @@ import os import sys -from snap_to_bucket import Ec2Handler -from snap_to_bucket import S3Handler -from snap_to_bucket import FsHandler +from snap_to_bucket.handlers import Ec2Handler +from snap_to_bucket.handlers import S3Handler +from snap_to_bucket.handlers import FsHandler class SnapToBucket: