From cff490e5934a029deb617552757e70a05a859ab3 Mon Sep 17 00:00:00 2001 From: tdruez <489057+tdruez@users.noreply.github.com> Date: Mon, 5 Apr 2021 16:43:25 +0200 Subject: [PATCH] 6 Add support for RPM-based distros for docker and rootfs images (#127) * Add minimal support for RPM distros #6 Signed-off-by: Philippe Ombredanne * Relax scancode-toolkit version requirements Signed-off-by: Philippe Ombredanne * Install scancode-toolkit[packages] for rpm support #6 Signed-off-by: Thomas Druez * Require newest RPM plugin and its deps Signed-off-by: Philippe Ombredanne * Update documentation for all OSes open is a macOS'ism Signed-off-by: Philippe Ombredanne * Require newest RPM plugin and its deps Signed-off-by: Philippe Ombredanne * Update documentation for all OSes open is a macOS'ism Signed-off-by: Philippe Ombredanne * Remove explicit dependency on rpm-inspector-rpm This is not needed as it comes with scancode-tk Signed-off-by: Philippe Ombredanne * Add changelog entry for RPM support #6 Signed-off-by: Thomas Druez Co-authored-by: Philippe Ombredanne --- CHANGELOG.rst | 3 +++ Dockerfile | 46 ++++++++++++++++++++++++++++++++++++--- docs/installation.rst | 8 ++++++- etc/requirements/base.txt | 4 ++-- scanpipe/pipes/rootfs.py | 7 ++++++ scanpipe/pipes/rpm.py | 32 +++++++++++++++++++++++++++ 6 files changed, 94 insertions(+), 6 deletions(-) create mode 100644 scanpipe/pipes/rpm.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 627247067..d24f91e21 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -3,6 +3,9 @@ ### v1.1.1 (unreleased) +- Add support for RPM-based distros for docker and rootfs images + https://github.com/nexB/scancode.io/issues/6 + - Add a compliance alert system based on license policies provided through a policies.yml file. The compliance alerts are computed from the license_expression and stored on the codebase resource. When the policy feature is enabled, the compliance diff --git a/Dockerfile b/Dockerfile index 01df5adb3..f66acbe6a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,50 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# http://nexb.com and https://github.com/nexB/scancode.io +# The ScanCode.io software is licensed under the Apache License version 2.0. +# Data generated with ScanCode.io is provided as-is without warranties. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode.io should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# +# ScanCode.io is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/nexB/scancode.io for support and download. + + FROM python:3.9 + +# Force unbuffered stdout and stderr (e.g. they are flushed to terminal immediately) ENV PYTHONUNBUFFERED 1 + +# Requirements as per https://scancode-toolkit.readthedocs.io/en/latest/getting-started/install.html +RUN apt-get update \ + && apt-get install -y \ + bzip2 \ + xz-utils \ + zlib1g \ + libxml2-dev \ + libxslt1-dev \ + libgomp1 \ + libsqlite3-0 \ + libgcrypt20 \ + libpopt0 \ + libzstd1 \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + RUN mkdir /opt/scancodeio/ RUN mkdir -p /var/scancodeio/static/ RUN mkdir -p /var/scancodeio/workspace/ -WORKDIR /opt/scancodeio/ -COPY etc/requirements/base.txt /opt/scancodeio/ -RUN pip install -r base.txt COPY . /opt/scancodeio/ +WORKDIR /opt/scancodeio/ RUN pip install . diff --git a/docs/installation.rst b/docs/installation.rst index ed278195e..53b85f165 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -22,6 +22,7 @@ Optional: ScanCode.io can also be run through a Docker image, refer to the :ref:`docker_image` chapter for details. + Local installation ------------------ @@ -40,6 +41,7 @@ install dependencies and create an environment file:: When ``PYTHON_EXE`` is not specified, the default ``python3`` executable is used. + Database -------- @@ -60,6 +62,7 @@ development as a single user:: https://docs.djangoproject.com/en/dev/ref/databases/#sqlite-notes for details. + Tests ----- @@ -67,6 +70,7 @@ Validate the installation by running the tests suite:: make test + Web Application --------------- @@ -74,13 +78,15 @@ A web application is available to create and manage your projects from a browser To start the local webserver and access the app:: make run - open http://127.0.0.1:8001/ + +Then open you web browser at visit: http://127.0.0.1:8001/ ------------------ .. note:: You are now ready to move onto the **Tutorials**: :ref:`scanpipe_tutorial_1`. + Upgrading --------- diff --git a/etc/requirements/base.txt b/etc/requirements/base.txt index 701cc8c33..a1185d389 100644 --- a/etc/requirements/base.txt +++ b/etc/requirements/base.txt @@ -24,10 +24,10 @@ redis==3.5.3 gunicorn==20.1.0 # Docker -container_inspector==3.1.2 +container_inspector>=3.1.2 # ScanCode-toolkit -scancode-toolkit==21.3.31 +scancode-toolkit[packages]==21.3.31 # Utilities XlsxWriter==1.3.8 diff --git a/scanpipe/pipes/rootfs.py b/scanpipe/pipes/rootfs.py index 72289d260..7c414df6a 100644 --- a/scanpipe/pipes/rootfs.py +++ b/scanpipe/pipes/rootfs.py @@ -33,6 +33,7 @@ from scanpipe import pipes from scanpipe.pipes import alpine from scanpipe.pipes import debian +from scanpipe.pipes import rpm logger = logging.getLogger(__name__) @@ -40,6 +41,12 @@ "alpine": alpine.package_getter, "debian": partial(debian.package_getter, distro="debian"), "ubuntu": partial(debian.package_getter, distro="ubuntu"), + "rhel": rpm.package_getter, + "centos": rpm.package_getter, + "fedora": rpm.package_getter, + "sles": rpm.package_getter, + "opensuse": rpm.package_getter, + "opensuse-tumbleweed": rpm.package_getter, } diff --git a/scanpipe/pipes/rpm.py b/scanpipe/pipes/rpm.py new file mode 100644 index 000000000..d2324b560 --- /dev/null +++ b/scanpipe/pipes/rpm.py @@ -0,0 +1,32 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# http://nexb.com and https://github.com/nexB/scancode.io +# The ScanCode.io software is licensed under the Apache License version 2.0. +# Data generated with ScanCode.io is provided as-is without warranties. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode.io should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# +# ScanCode.io is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/nexB/scancode.io for support and download. + +from packagedcode import rpm + + +def package_getter(root_dir, detect_licenses=True, **kwargs): + """ + Yield installed package objects. + """ + packages = rpm.get_installed_packages(root_dir, detect_licenses=detect_licenses) + for package in packages: + yield package.purl, package