diff --git a/setup.py b/setup.py index 05618a9d191..4387f62be55 100644 --- a/setup.py +++ b/setup.py @@ -193,6 +193,7 @@ def read(*names, **kwargs): 'typecode_libmagic', # packagedcode + 'debut >= 0.9.4', 'pefile >= 2018.8.8', 'pymaven_patch >= 0.2.8', 'requests >= 2.7.0, < 3.0.0', diff --git a/src/packagedcode/debian.py b/src/packagedcode/debian.py new file mode 100644 index 00000000000..8a46c57b974 --- /dev/null +++ b/src/packagedcode/debian.py @@ -0,0 +1,126 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# http://nexb.com and https://github.com/nexB/scancode-toolkit/ +# The ScanCode software is licensed under the Apache License version 2.0. +# Data generated with ScanCode require an acknowledgment. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# When you publish or redistribute any data created with ScanCode or any ScanCode +# derivative work, you must accompany this data with the following acknowledgment: +# +# Generated with ScanCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# ScanCode is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/nexB/scancode-toolkit/ for support and download. + +from __future__ import absolute_import +from __future__ import print_function +from __future__ import unicode_literals + +from collections import OrderedDict +import logging + +import attr +from six import string_types +from debut import debcon + +from commoncode import filetype +from commoncode import fileutils +from packagedcode import models + +""" +Handle FreeBSD ports +per https://www.freebsd.org/cgi/man.cgi?pkg-create(8) +""" + +TRACE = False + +logger = logging.getLogger(__name__) + +if TRACE: + import sys + logging.basicConfig(level=logging.DEBUG, stream=sys.stdout) + logger.setLevel(logging.DEBUG) + + +@attr.s() +class DebianPackage(models.Package): + metafiles = ('status',) + default_type = 'debian' + +# @classmethod +# def recognize(cls, location): +# yield parse(location) + + +def is_debian_status_file(location): + return (filetype.is_file(location) + and fileutils.file_name(location).lower() == 'status') + + +def parse_status_file(location): + """ + Yield Debian Package objects from a dpkg `status` file or None. + """ + if not is_debian_status_file(location): + return + + for debian_pkg_data in debcon.get_paragraphs_data_from_file(location): + yield build_package(debian_pkg_data) + + +def build_package(package_data): + """ + Return a Package object from a package_data mapping (from a dpkg status file) + or None. + """ + # construct the package + package = DebianPackage() + + # add debian-specific package 'qualifiers' + qualifiers = OrderedDict([ + ('arch', package_data.get('architecture')), + ]) + package.qualifiers = qualifiers + + # mapping of top level `status` file items to the Package object field name + plain_fields = [ + ('description', 'description'), + ('homepage', 'homepage_url'), + ('installed-size', 'size'), + ('package', 'name'), + ('version', 'version'), + ('maintainer', 'maintainer'), + ] + + for source, target in plain_fields: + value = package_data.get(source) + if value: + if isinstance(value, string_types): + value = value.strip() + if value: + setattr(package, target, value) + + # mapping of top level `status` file items to a function accepting as + # arguments the package.json element value and returning an iterable of key, + # values Package Object to update + field_mappers = [ + #('depends', dependency_mapper), + ] + + for source, func in field_mappers: + logger.debug('parse: %(source)r, %(func)r' % locals()) + value = package_data.get(source) or None + if value: + func(value, package) + + return package diff --git a/tests/packagedcode/data/debian/basic/status b/tests/packagedcode/data/debian/basic/status new file mode 100644 index 00000000000..59b52f5605c --- /dev/null +++ b/tests/packagedcode/data/debian/basic/status @@ -0,0 +1,136 @@ +Package: libncurses5 +Status: install ok installed +Priority: optional +Section: libs +Installed-Size: 283 +Maintainer: Ubuntu Developers +Architecture: amd64 +Multi-Arch: same +Source: ncurses +Version: 6.1-1ubuntu1.18.04 +Depends: libtinfo5 (= 6.1-1ubuntu1.18.04), libc6 (>= 2.14) +Recommends: libgpm2 +Description: shared libraries for terminal handling + The ncurses library routines are a terminal-independent method of + updating character screens with reasonable optimization. + . + This package contains the shared libraries necessary to run programs + compiled with ncurses. +Homepage: https://invisible-island.net/ncurses/ +Original-Maintainer: Craig Small + +Package: libcom-err2 +Status: install ok installed +Priority: optional +Section: libs +Installed-Size: 87 +Maintainer: Ubuntu Developers +Architecture: amd64 +Multi-Arch: same +Source: e2fsprogs +Version: 1.44.1-1ubuntu1.1 +Replaces: libcomerr2 (<< 1.43.9-1~) +Provides: libcomerr2 (= 1.44.1-1ubuntu1.1) +Depends: libc6 (>= 2.17) +Breaks: libcomerr2 (<< 1.43.9-1~) +Description: common error description library + libcomerr is an attempt to present a common error-handling mechanism to + manipulate the most common form of error code in a fashion that does not + have the problems identified with mechanisms commonly in use. +Homepage: http://e2fsprogs.sourceforge.net +Original-Maintainer: Theodore Y. Ts'o + +Package: libapt-pkg5.0 +Status: install ok installed +Priority: optional +Section: libs +Installed-Size: 3139 +Maintainer: Ubuntu Developers +Architecture: amd64 +Multi-Arch: same +Source: apt +Version: 1.6.11 +Provides: libapt-pkg (= 1.6.11) +Depends: libbz2-1.0, libc6 (>= 2.27), libgcc1 (>= 1:3.0), liblz4-1 (>= 0.0~r127), liblzma5 (>= 5.1.1alpha+20120614), libstdc++6 (>= 5.2), libsystemd0 (>= 221), libudev1 (>= 183), libzstd1 (>= 1.3.2), zlib1g (>= 1:1.2.2.3) +Recommends: apt (>= 1.6.11) +Breaks: appstream (<< 0.9.0-3~), apt (<< 1.1~exp14), libapt-inst1.5 (<< 0.9.9~) +Description: package management runtime library + This library provides the common functionality for searching and + managing packages as well as information about packages. + Higher-level package managers can depend upon this library. + . + This includes: + * retrieval of information about packages from multiple sources + * retrieval of packages and all dependent packages + needed to satisfy a request either through an internal + solver or by interfacing with an external one + * authenticating the sources and validating the retrieved data + * installation and removal of packages in the system + * providing different transports to retrieve data over cdrom, ftp, + http(s), rsh as well as an interface to add more transports like + tor+http(s) (apt-transport-tor). +Original-Maintainer: APT Development Team + +Package: libaudit1 +Status: install ok installed +Priority: optional +Section: libs +Installed-Size: 147 +Maintainer: Ubuntu Developers +Architecture: amd64 +Multi-Arch: same +Source: audit +Version: 1:2.8.2-1ubuntu1 +Depends: libaudit-common (>= 1:2.8.2-1ubuntu1), libc6 (>= 2.14), libcap-ng0 +Description: Dynamic library for security auditing + The audit-libs package contains the dynamic libraries needed for + applications to use the audit framework. It is used to monitor systems for + security related events. +Homepage: https://people.redhat.com/sgrubb/audit/ +Original-Maintainer: Laurent Bigonville + +Package: perl-base +Essential: yes +Status: install ok installed +Priority: required +Section: perl +Installed-Size: 7826 +Maintainer: Ubuntu Developers +Architecture: amd64 +Source: perl +Version: 5.26.1-6ubuntu0.3 +Replaces: libfile-path-perl (<< 2.12.01), libfile-temp-perl (<< 0.2304), libio-socket-ip-perl (<< 0.38), libscalar-list-utils-perl (<< 1:1.46.02), libsocket-perl (<< 2.020.03), libxsloader-perl (<< 0.27), perl (<< 5.10.1-12), perl-modules (<< 5.20.1-3) +Provides: libfile-path-perl, libfile-temp-perl, libio-socket-ip-perl, libscalar-list-utils-perl, libsocket-perl, libxsloader-perl, perlapi-5.26.0, perlapi-5.26.1 +Pre-Depends: libc6 (>= 2.23), dpkg (>= 1.17.17) +Suggests: perl +Breaks: amanda-common (<< 1:3.3.9-2), autoconf2.13 (<< 2.13-45), backuppc (<< 3.3.1-2), debconf (<< 1.5.61), dh-haskell (<< 0.3), intltool (<< 0.51.0-4), libalien-wxwidgets-perl (<< 0.65+dfsg-2), libanyevent-perl (<< 7.070-2), libcommon-sense-perl (<< 3.72-2~), libe +xception-class-perl (<< 1.42), libfile-path-perl (<< 2.12.01), libfile-spec-perl (<< 3.6700), libfile-temp-perl (<< 0.2304), libgtk2-perl-doc (<< 2:1.2491-4), libio-socket-ip-perl (<< 0.38), libjcode-perl (<< 2.13-3), libmarc-charset-perl (<< 1.2), libsbuild-perl (< +< 0.67.0-1), libscalar-list-utils-perl (<< 1:1.46.02), libsocket-perl (<< 2.020.03), libxsloader-perl (<< 0.27), mailagent (<< 1:3.1-81-2), pdl (<< 1:2.007-4), perl (<< 5.26.1~), perl-modules (<< 5.26.1~), slic3r (<< 1.2.9+dfsg-6.1), slic3r-prusa (<< 1.37.0+dfsg-1.1 +), texinfo (<< 6.1.0.dfsg.1-8) +Conflicts: defoma (<< 0.11.12), doc-base (<< 0.10.3), mono-gac (<< 2.10.8.1-3), safe-rm (<< 0.8), update-inetd (<< 4.41) +Description: minimal Perl system + Perl is a scripting language used in many system scripts and utilities. + . + This package provides a Perl interpreter and the small subset of the + standard run-time library required to perform basic tasks. For a full + Perl installation, install "perl" (and its dependencies, "perl-modules-5.26" + and "perl-doc"). +Homepage: http://dev.perl.org/perl5/ +Original-Maintainer: Niko Tyni + +Package: libudev1 +Status: install ok installed +Priority: optional +Section: libs +Installed-Size: 227 +Maintainer: Ubuntu Developers +Architecture: amd64 +Multi-Arch: same +Source: systemd +Version: 237-3ubuntu10.22 +Depends: libc6 (>= 2.25) +Description: libudev shared library + This library provides access to udev device information. +Homepage: https://www.freedesktop.org/wiki/Software/systemd +Original-Maintainer: Debian systemd Maintainers + diff --git a/tests/packagedcode/data/debian/basic/status.expected b/tests/packagedcode/data/debian/basic/status.expected new file mode 100644 index 00000000000..dfe6ffbf25c --- /dev/null +++ b/tests/packagedcode/data/debian/basic/status.expected @@ -0,0 +1,224 @@ +[ + { + "type": "debian", + "namespace": null, + "name": "libncurses5", + "version": "6.1-1ubuntu1.18.04", + "qualifiers": { + "arch": "amd64" + }, + "subpath": null, + "primary_language": null, + "description": "shared libraries for terminal handling\n The ncurses library routines are a terminal-independent method of\n updating character screens with reasonable optimization.\n .\n This package contains the shared libraries necessary to run programs\n compiled with ncurses.", + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": "https://invisible-island.net/ncurses/", + "download_url": null, + "size": "283", + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": null, + "notice_text": null, + "root_path": null, + "dependencies": [], + "contains_source_code": null, + "source_packages": [], + "purl": "pkg:debian/libncurses5@6.1-1ubuntu1.18.04?arch=amd64", + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null + }, + { + "type": "debian", + "namespace": null, + "name": "libcom-err2", + "version": "1.44.1-1ubuntu1.1", + "qualifiers": { + "arch": "amd64" + }, + "subpath": null, + "primary_language": null, + "description": "common error description library\n libcomerr is an attempt to present a common error-handling mechanism to\n manipulate the most common form of error code in a fashion that does not\n have the problems identified with mechanisms commonly in use.", + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": "http://e2fsprogs.sourceforge.net", + "download_url": null, + "size": "87", + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": null, + "notice_text": null, + "root_path": null, + "dependencies": [], + "contains_source_code": null, + "source_packages": [], + "purl": "pkg:debian/libcom-err2@1.44.1-1ubuntu1.1?arch=amd64", + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null + }, + { + "type": "debian", + "namespace": null, + "name": "libapt-pkg5.0", + "version": "1.6.11", + "qualifiers": { + "arch": "amd64" + }, + "subpath": null, + "primary_language": null, + "description": "package management runtime library\n This library provides the common functionality for searching and\n managing packages as well as information about packages.\n Higher-level package managers can depend upon this library.\n .\n This includes:\n * retrieval of information about packages from multiple sources\n * retrieval of packages and all dependent packages\n needed to satisfy a request either through an internal\n solver or by interfacing with an external one\n * authenticating the sources and validating the retrieved data\n * installation and removal of packages in the system\n * providing different transports to retrieve data over cdrom, ftp,\n http(s), rsh as well as an interface to add more transports like\n tor+http(s) (apt-transport-tor).", + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": null, + "size": "3139", + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": null, + "notice_text": null, + "root_path": null, + "dependencies": [], + "contains_source_code": null, + "source_packages": [], + "purl": "pkg:debian/libapt-pkg5.0@1.6.11?arch=amd64", + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null + }, + { + "type": "debian", + "namespace": null, + "name": "libaudit1", + "version": "1:2.8.2-1ubuntu1", + "qualifiers": { + "arch": "amd64" + }, + "subpath": null, + "primary_language": null, + "description": "Dynamic library for security auditing\n The audit-libs package contains the dynamic libraries needed for\n applications to use the audit framework. It is used to monitor systems for\n security related events.", + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": "https://people.redhat.com/sgrubb/audit/", + "download_url": null, + "size": "147", + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": null, + "notice_text": null, + "root_path": null, + "dependencies": [], + "contains_source_code": null, + "source_packages": [], + "purl": "pkg:debian/libaudit1@1:2.8.2-1ubuntu1?arch=amd64", + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null + }, + { + "type": "debian", + "namespace": null, + "name": "perl-base", + "version": "5.26.1-6ubuntu0.3", + "qualifiers": { + "arch": "amd64" + }, + "subpath": null, + "primary_language": null, + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": null, + "size": "7826", + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": null, + "notice_text": null, + "root_path": null, + "dependencies": [], + "contains_source_code": null, + "source_packages": [], + "purl": "pkg:debian/perl-base@5.26.1-6ubuntu0.3?arch=amd64", + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null + }, + { + "type": "debian", + "namespace": null, + "name": "libudev1", + "version": "237-3ubuntu10.22", + "qualifiers": { + "arch": "amd64" + }, + "subpath": null, + "primary_language": null, + "description": "libudev shared library\n This library provides access to udev device information.", + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": "https://www.freedesktop.org/wiki/Software/systemd", + "download_url": null, + "size": "227", + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": null, + "notice_text": null, + "root_path": null, + "dependencies": [], + "contains_source_code": null, + "source_packages": [], + "purl": "pkg:debian/libudev1@237-3ubuntu10.22?arch=amd64", + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null + } +] \ No newline at end of file diff --git a/tests/packagedcode/data/debian/not-a-status-file b/tests/packagedcode/data/debian/not-a-status-file new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/packagedcode/test_debian.py b/tests/packagedcode/test_debian.py new file mode 100644 index 00000000000..da0b3807afd --- /dev/null +++ b/tests/packagedcode/test_debian.py @@ -0,0 +1,47 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# http://nexb.com and https://github.com/nexB/scancode-toolkit/ +# The ScanCode software is licensed under the Apache License version 2.0. +# Data generated with ScanCode require an acknowledgment. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# When you publish or redistribute any data created with ScanCode or any ScanCode +# derivative work, you must accompany this data with the following acknowledgment: +# +# Generated with ScanCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# ScanCode is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/nexB/scancode-toolkit/ for support and download. + +from __future__ import absolute_import +from __future__ import print_function +from __future__ import unicode_literals + +import os.path + +from packagedcode import debian +from packages_test_utils import PackageTester + + +class TestDebian(PackageTester): + test_data_dir = os.path.join(os.path.dirname(__file__), 'data') + + def test_parse_status_file_not_a_status_file(self): + test_file = self.get_test_loc('debian/not-a-status-file') + test_packages = list(debian.parse_status_file(test_file)) + self.assertEqual([], test_packages) + + def test_parse_status_file_basic(self): + test_file = self.get_test_loc('debian/basic/status') + expected_loc = self.get_test_loc('debian/basic/status.expected') + packages = list(debian.parse_status_file(test_file)) + self.check_packages(packages, expected_loc, regen=False) diff --git a/thirdparty/attrs-19.3.0-py2.py3-none-any.whl b/thirdparty/attrs-19.3.0-py2.py3-none-any.whl new file mode 100644 index 00000000000..db90cdc7c2c Binary files /dev/null and b/thirdparty/attrs-19.3.0-py2.py3-none-any.whl differ diff --git a/thirdparty/debut-0.9.4-py2.py3-none-any.whl b/thirdparty/debut-0.9.4-py2.py3-none-any.whl new file mode 100644 index 00000000000..8fc5d57084e Binary files /dev/null and b/thirdparty/debut-0.9.4-py2.py3-none-any.whl differ