Merge pull request #114 from nexB/purl2sym-metadata-4

Add metadata support for openssl, erofs-utils

CHANGELOG.rst

@@ -2,6 +2,39 @@ Changelog
 =========
 
 
+v0.5.0
+-------
+- FetchCode now supports retrieving package info for following generic packages: 
+    * pkg:generic/linux
+    * pkg:generic/mtd-utils
+    * pkg:generic/barebox
+    * pkg:generic/e2fsprogs
+    * pkg:generic/udhcp
+    * pkg:generic/miniupnpc
+    * pkg:generic/miniupnpd
+    * pkg:generic/minissdpd
+    * pkg:generic/erofs-utils
+    * pkg:openssl/openssl
+
+- FetchCode also supports retrieving package info for packages hosted on GitHub specifically.
+    * pkg:github/avahi/avahi
+    * pkg:github/bestouff/genext2fs
+    * pkg:github/dosfstools/dosfstools
+    * pkg:github/google/brotli
+    * pkg:github/hewlettpackard/wireless-tools
+    * pkg:github/inotify-tools/inotify-tools
+    * pkg:github/libbpf/bpftool
+    * pkg:github/llvm/llvm-project
+    * pkg:github/nixos/nix
+    * pkg:github/plougher/squashfs-tools
+    * pkg:github/pupnp/pupnp
+    * pkg:github/python/cpython
+    * pkg:github/rpm-software-management/rpm
+    * pkg:github/shadow-maint/shadow
+    * pkg:github/sqlite/sqlite
+    * pkg:github/u-boot/u-boot
+
+
 v0.4.0
 -------
 - FetchCode now supports retrieving package info for following generic packages: 

src/fetchcode/package.py

@@ -27,8 +27,11 @@
 
 from fetchcode.package_util import GITHUB_SOURCE_BY_PACKAGE
 from fetchcode.package_util import IPKG_RELEASES
+from fetchcode.package_util import UDHCP_RELEASES
+from fetchcode.package_util import ErofsUtilsGitHubSource
 from fetchcode.package_util import GitHubSource
 from fetchcode.package_util import MiniupnpPackagesGitHubSource
+from fetchcode.package_util import OpenSSLGitHubSource
 from fetchcode.packagedcode_models import Package
 from fetchcode.utils import get_response
 
@@ -242,7 +245,25 @@ def get_github_data_for_miniupnp(purl):
     )
 
 
-@router.route("pkg:generic/erofs-utils.*",)
+@router.route(
+    "pkg:openssl/openssl.*",
+)
+def get_github_data_for_openssl(purl):
+    """
+    Yield `Package` object for OpenSSL package from GitHub.
+    """
+    generic_purl = PackageURL.from_string(purl)
+    github_repo_purl = PackageURL(
+        type="github",
+        namespace="openssl",
+        name="openssl",
+        version=generic_purl.version,
+    )
+
+    return OpenSSLGitHubSource.get_package_info(github_repo_purl)
+
+
+@router.route("pkg:generic/erofs-utils.*")
 def get_github_data_for_erofs_utils(purl):
     """
     Yield `Package` object for erofs-utils package from GitHub.
@@ -255,7 +276,7 @@ def get_github_data_for_erofs_utils(purl):
         version=generic_purl.version,
     )
 
-    return GitHubSource.get_package_info(github_repo_purl)
+    return ErofsUtilsGitHubSource.get_package_info(github_repo_purl)
 
 
 @router.route("pkg:bitbucket/.*")
@@ -384,6 +405,37 @@ def get_package_info(cls, package_url):
             )
 
 
+# The udhcp is no longer maintained as a standalone project.
+# It has been fully integrated into busybox.
+class UdhcpDirectoryListedSource(DirectoryListedSource):
+    source_url = (
+        "https://web.archive.org/web/20021209021312/http://udhcp.busybox.net/source/"
+    )
+
+    @classmethod
+    def get_package_info(cls, package_url):
+
+        version = package_url.version
+        if version and version in UDHCP_RELEASES:
+            archive = UDHCP_RELEASES[version]
+            yield Package(
+                homepage_url=cls.source_url,
+                download_url=archive["url"],
+                release_date=archive["date"],
+                **package_url.to_dict(),
+            )
+
+        else:
+            for version, data in UDHCP_RELEASES.items():
+                purl = PackageURL(type="generic", name="udhcp", version=version)
+                yield Package(
+                    homepage_url=cls.source_url,
+                    download_url=data["url"],
+                    release_date=data["date"],
+                    **purl.to_dict(),
+                )
+
+
 class IpkgDirectoryListedSource(DirectoryListedSource):
     source_url = "https://web.archive.org/web/20090326020239/http://handhelds.org/download/packages/ipkg/"
     is_nested = False
@@ -612,6 +664,7 @@ class BareboxDirectoryListedSource(DirectoryListedSource):
     is_nested = False
     ignored_files_and_dir = []
 
+
 class LinuxDirectoryListedSource(DirectoryListedSource):
     source_url = "https://cdn.kernel.org/pub/linux/kernel/"
     # Source archive ex: linux-1.2.3.tar.gz
@@ -631,8 +684,11 @@ class LinuxDirectoryListedSource(DirectoryListedSource):
         "uemacs/",
     ]
 
+
 class E2fsprogsDirectoryListedSource(DirectoryListedSource):
-    source_url = "https://mirrors.edge.kernel.org/pub/linux/kernel/people/tytso/e2fsprogs/"
+    source_url = (
+        "https://mirrors.edge.kernel.org/pub/linux/kernel/people/tytso/e2fsprogs/"
+    )
     # Source archive ex: e2fsprogs-1.2.3.tar.gz
     source_archive_regex = re.compile(r"^(e2fsprogs-)(?P<version>[\w.-]*)(.tar.gz)$")
     is_nested = True
@@ -665,6 +721,7 @@ class E2fsprogsDirectoryListedSource(DirectoryListedSource):
     "pkg:generic/barebox.*",
     "pkg:generic/linux.*",
     "pkg:generic/e2fsprogs.*",
+    "pkg:generic/udhcp.*",
 ]
 
 DIR_LISTED_SOURCE_BY_PACKAGE_NAME = {
@@ -692,6 +749,7 @@ class E2fsprogsDirectoryListedSource(DirectoryListedSource):
     "barebox": BareboxDirectoryListedSource,
     "linux": LinuxDirectoryListedSource,
     "e2fsprogs": E2fsprogsDirectoryListedSource,
+    "udhcp": UdhcpDirectoryListedSource,
 }
 
 

src/fetchcode/package_util.py

@@ -14,9 +14,6 @@
 # CONDITIONS OF ANY KIND, either express or implied. See the License for the
 # specific language governing permissions and limitations under the License.
 
-# Since there will be no new releases of ipkg, it's better to
-# store them in a dictionary rather than fetching them every time.
-
 import dataclasses
 import re
 
@@ -211,6 +208,54 @@ class RpmGitHubSource(GitHubSource):
 }
 
 
+class OpenSSLGitHubSource(GitHubSource):
+    version_regex = re.compile(r"(OpenSSL_|openssl-)(?P<version>.+)")
+    ignored_tag_regex = None
+
+    @classmethod
+    def get_package_info(cls, gh_purl):
+
+        packages = get_github_packages(
+            gh_purl,
+            cls.version_regex,
+            cls.ignored_tag_regex,
+            cls.get_default_package(gh_purl),
+        )
+
+        for package in packages:
+            package_dict = package.to_dict()
+            package_dict["type"] = "openssl"
+            package_dict["namespace"] = None
+            package_dict["name"] = "openssl"
+            package_dict["version"] = package_dict["version"].replace("_", ".")
+
+            yield package_from_dict(package_dict)
+
+
+class ErofsUtilsGitHubSource(GitHubSource):
+    version_regex = None
+    ignored_tag_regex = None
+
+    @classmethod
+    def get_package_info(cls, gh_purl):
+
+        packages = get_github_packages(
+            gh_purl,
+            cls.version_regex,
+            cls.ignored_tag_regex,
+            cls.get_default_package(gh_purl),
+        )
+
+        for package in packages:
+            package_dict = package.to_dict()
+            package_dict["type"] = "generic"
+            package_dict["namespace"] = None
+            package_dict["name"] = "erofs-utils"
+            package_dict["version"] = package_dict["version"].replace("_", ".")
+
+            yield package_from_dict(package_dict)
+
+
 class MiniupnpPackagesGitHubSource(GitHubSource):
     version_regex = None
     ignored_tag_regex = None
@@ -231,14 +276,53 @@ def get_package_info(cls, gh_purl, package_name):
 
         for package in packages:
             package_dict = package.to_dict()
+            package_dict["type"] = "generic"
             package_dict["namespace"] = None
             package_dict["name"] = package_name
-            package_dict["type"] = "generic"
             package_dict["version"] = package_dict["version"].replace("_", ".")
 
             yield package_from_dict(package_dict)
 
 
+# Archive source https://web.archive.org/web/20021209021312/http://udhcp.busybox.net/source/
+UDHCP_RELEASES = {
+    "0.9.1": {
+        "url": "https://web.archive.org/web/20021209021312/http://udhcp.busybox.net/source//udhcp-0.9.1.tar.gz",
+        "date": "2001-08-10T20:17:00",
+    },
+    "0.9.2": {
+        "url": "https://web.archive.org/web/20021209021312/http://udhcp.busybox.net/source//udhcp-0.9.2.tar.gz",
+        "date": "2001-08-10T20:17:00",
+    },
+    "0.9.3": {
+        "url": "https://web.archive.org/web/20021209021312/http://udhcp.busybox.net/source//udhcp-0.9.3.tar.gz",
+        "date": "2001-08-20T18:23:00",
+    },
+    "0.9.4": {
+        "url": "https://web.archive.org/web/20021209021312/http://udhcp.busybox.net/source//udhcp-0.9.4.tar.gz",
+        "date": "2001-08-27T15:41:00",
+    },
+    "0.9.5": {
+        "url": "https://web.archive.org/web/20021209021312/http://udhcp.busybox.net/source//udhcp-0.9.5.tar.gz",
+        "date": "2001-09-14T18:19:00",
+    },
+    "0.9.6": {
+        "url": "https://web.archive.org/web/20021209021312/http://udhcp.busybox.net/source//udhcp-0.9.6.tar.gz",
+        "date": "2001-10-01T13:38:00",
+    },
+    "0.9.7": {
+        "url": "https://web.archive.org/web/20021209021312/http://udhcp.busybox.net/source//udhcp-0.9.7.tar.gz",
+        "date": "2002-05-27T00:48:00",
+    },
+    "0.9.8": {
+        "url": "https://web.archive.org/web/20021209021312/http://udhcp.busybox.net/source//udhcp-0.9.8.tar.gz",
+        "date": "2002-10-31T12:10:00",
+    },
+}
+
+
+# Since there will be no new releases of ipkg, it's better to
+# store them in a dictionary rather than fetching them every time.
 IPKG_RELEASES = {
     "0.99.88": {
         "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg-0.99.88.tar.gz",