Merge pull request #107 from marcelotrevisani/use-spdx-org

Using license identifier from spdx.org instead of opensource.org

grayskull/license/discovery.py

@@ -29,33 +29,37 @@ class ShortLicense:
 
 
 @lru_cache(maxsize=10)
-def get_all_licenses_from_opensource() -> List:
-    """Get all licenses available on opensource.org
+def get_all_licenses_from_spdx() -> List:
+    """Get all licenses available on spdx.org
 
-    :return: List with all licenses information on opensource.org
+    :return: List with all licenses information on spdx.org
     """
-    response = requests.get(url="https://api.opensource.org/licenses", timeout=5)
+    response = requests.get(url="https://spdx.org/licenses/licenses.json", timeout=5)
     log.debug(
-        f"Response from api.opensource. Status code:{response.status_code},"
+        f"Response from spdx.org. Status code:{response.status_code},"
         f" response: {response}"
     )
     if response.status_code != 200:
         raise HTTPError(
-            f"It was not possible to communicate with opensource api.\n{response.text}"
+            f"It was not possible to communicate with spdx api.\n{response.text}"
         )
-    print(f"{Fore.LIGHTBLACK_EX}Recovering license info from opensource.org ...")
-    return response.json()
+    print(f"{Fore.LIGHTBLACK_EX}Recovering license info from spdx.org ...")
+    return [
+        l
+        for l in response.json()["licenses"]
+        if not l.get("isDeprecatedLicenseId", False)
+    ]
 
 
 def match_license(name: str) -> dict:
     """Match if the given license name matches any license present on
-    opensource.org
+    spdx.org
 
     :param name: License name
     :return: Information of the license matched
     """
-    all_licenses = get_all_licenses_from_opensource()
-    name = name.strip()
+    all_licenses = get_all_licenses_from_spdx()
+    name = re.sub(r"\s+license\s*", "", name.strip(), flags=re.IGNORECASE)
 
     best_match = process.extractOne(name, _get_all_license_choice(all_licenses))
     log.info(f"Best match for license {name} was {best_match}")
@@ -70,10 +74,7 @@ def get_short_license_id(name: str) -> str:
     :return: short identifier (spdx) for the given license name
     """
     recipe_license = match_license(name)
-    for identifier in recipe_license["identifiers"]:
-        if identifier["scheme"].lower() == "spdx":
-            return identifier["identifier"]
-    return recipe_license["id"]
+    return recipe_license["licenseId"]
 
 
 def _get_license(license_id: str, all_licenses: List) -> dict:
@@ -97,16 +98,37 @@ def _get_all_names_from_api(one_license: dict) -> List:
     result = set()
     if one_license["name"]:
         result.add(one_license["name"])
-    if one_license["id"]:
-        result.add(one_license["id"])
-    for lc in one_license["identifiers"]:
-        if lc["scheme"] == "spdx":
-            result.add(lc["identifier"])
-            break
-    result = result.union({l["name"] for l in one_license["other_names"]})
+    if one_license["licenseId"]:
+        result.add(one_license["licenseId"])
+    other_names = get_other_names_from_opensource(one_license["licenseId"])
+    result.update(other_names)
     return list(result)
 
 
+def get_other_names_from_opensource(license_spdx: str) -> List:
+    lic = get_opensource_license(license_spdx)
+    return [l["name"] for l in lic.get("other_names", [])]
+
+
+def get_opensource_license(license_spdx: str) -> dict:
+    opensource = get_opensource_license_data()
+    for lic in opensource:
+        if lic["id"] == license_spdx:
+            return lic
+        for _id in lic["identifiers"]:
+            if _id["scheme"].lower() == "spdx" and license_spdx == _id["identifier"]:
+                return lic
+    return {}
+
+
+@lru_cache(maxsize=10)
+def get_opensource_license_data() -> List:
+    response = requests.get(url=f"https://api.opensource.org/licenses/", timeout=5)
+    if response.status_code != 200:
+        return []
+    return response.json()
+
+
 def _get_all_license_choice(all_licenses: List) -> List:
     """Function responsible to get the whole licenses name
 

tests/license/test_discovery.py

@@ -9,8 +9,10 @@
     _get_all_names_from_api,
     _get_api_github_url,
     _get_license,
-    get_all_licenses_from_opensource,
+    get_all_licenses_from_spdx,
     get_license_type,
+    get_opensource_license_data,
+    get_other_names_from_opensource,
     get_short_license_id,
     match_license,
     search_license_api_github,
@@ -25,91 +27,60 @@ def license_pytest_path(data_dir) -> str:
 
 
 @fixture
-def opensource_license_mit() -> List:
+def spdx_org_license_mit() -> List:
     return [
         {
-            "id": "MIT",
-            "identifiers": [
-                {"identifier": "MIT", "scheme": "DEP5"},
-                {"identifier": "Expat", "scheme": "DEP5"},
-                {"identifier": "MIT", "scheme": "SPDX"},
-                {
-                    "identifier": "License :: OSI Approved :: MIT License",
-                    "scheme": "Trove",
-                },
-            ],
-            "links": [
-                {
-                    "note": "tl;dr legal",
-                    "url": "https://tldrlegal.com/license/mit-license",
-                },
-                {
-                    "note": "Wikipedia page",
-                    "url": "https://en.wikipedia.org/wiki/MIT_License",
-                },
-                {"note": "OSI Page", "url": "https://opensource.org/licenses/mit"},
-            ],
-            "name": "MIT/Expat License",
-            "other_names": [
-                {
-                    "name": "MIT",
-                    "note": "Because MIT has used many licenses for software, "
-                    "the Free Software Foundation considers MIT License"
-                    " ambiguous. The MIT License published on the OSI"
-                    " site is the same as the Expat License.",
-                },
-                {
-                    "name": "Expat",
-                    "note": "Because MIT has used many licenses for software,"
-                    " the Free Software Foundation considers MIT License"
-                    " ambiguous. The MIT License published on the OSI site"
-                    " is the same as the Expat License.",
-                },
-            ],
-            "superseded_by": None,
-            "keywords": ["osi-approved", "popular", "permissive"],
-            "text": [
-                {
-                    "media_type": "text/html",
-                    "title": "HTML",
-                    "url": "https://opensource.org/licenses/mit",
-                }
-            ],
+            "reference": "./MIT.html",
+            "isDeprecatedLicenseId": False,
+            "isFsfLibre": True,
+            "detailsUrl": "http://spdx.org/licenses/MIT.json",
+            "referenceNumber": "256",
+            "name": "MIT License",
+            "licenseId": "MIT",
+            "seeAlso": ["https://opensource.org/licenses/MIT"],
+            "isOsiApproved": True,
         }
     ]
 
 
 def test_match_license():
-    assert match_license("MIT License")["id"] == "MIT"
-    assert match_license("Expat")["id"] == "MIT"
+    assert match_license("MIT License")["licenseId"] == "MIT"
+    assert match_license("Expat")["licenseId"] == "MIT"
 
 
-def test_get_all_licenses_from_opensource():
-    assert len(get_all_licenses_from_opensource()) >= 88
-    assert get_all_licenses_from_opensource()[0]["id"]
+def test_get_all_licenses_from_spdx():
+    assert len(get_all_licenses_from_spdx()) > 300
+
+
+def test_get_opensource_license_data():
+    assert len(get_opensource_license_data()) >= 50
 
 
 def test_short_license_id():
     assert get_short_license_id("MIT License") == "MIT"
     assert get_short_license_id("Expat") == "MIT"
-    assert get_short_license_id("GPL 2.0") == "GPL-2.0"
+    assert get_short_license_id("GPL 2.0") == "GPL-2.0-only"
     assert get_short_license_id("2-Clause BSD License") == "BSD-2-Clause"
     assert get_short_license_id("3-Clause BSD License") == "BSD-3-Clause"
 
 
-def test_get_license(opensource_license_mit):
-    assert _get_license("MIT", opensource_license_mit) == opensource_license_mit[0]
+def test_get_other_names_from_opensource():
+    assert sorted(get_other_names_from_opensource("MIT")) == sorted(["MIT", "Expat"])
+
+
+def test_get_license(spdx_org_license_mit):
+    assert _get_license("MIT", spdx_org_license_mit) == spdx_org_license_mit[0]
 
 
-def test_get_all_names_from_api(opensource_license_mit):
-    assert sorted(_get_all_names_from_api(opensource_license_mit[0])) == sorted(
-        ["Expat", "MIT", "MIT/Expat License"]
+def test_get_all_names_from_api(spdx_org_license_mit):
+    assert sorted(_get_all_names_from_api(spdx_org_license_mit[0])) == sorted(
+        ["Expat", "MIT", "MIT License"]
     )
 
 
-def test_get_all_license_choice(opensource_license_mit):
-    assert sorted(_get_all_license_choice(opensource_license_mit)) == sorted(
-        ["Expat", "MIT", "MIT/Expat License"]
+def test_get_all_license_choice(spdx_org_license_mit):
+    assert sorted(_get_all_license_choice(spdx_org_license_mit)) == sorted(
+        ["Expat", "MIT", "MIT License"]
     )