perf: pinned python dependency count check

openedx · Oct 2, 2024 · c45d135 · c45d135
1 parent 98b78ad
commit c45d135
Show file tree

Hide file tree

Showing 19 changed files with 314 additions and 307 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -13,8 +13,8 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ubuntu-20.04]
-        python-version: ['3.8', '3.12']
+        os: [ubuntu-latest]
+        python-version: ['3.12']
         toxenv: [python, quality]
 
     steps:

diff --git a/pylintrc b/pylintrc
@@ -64,7 +64,7 @@
 # SERIOUSLY.
 #
 # ------------------------------
-# Generated by edx-lint version: 5.2.5
+# Generated by edx-lint version: 5.3.6
 # ------------------------------
 [MASTER]
 ignore = ,.git,.tox
@@ -259,6 +259,7 @@ enable =
 	useless-suppression,
 disable = 
 	bad-indentation,
+	broad-exception-raised,
 	consider-using-f-string,
 	duplicate-code,
 	file-ignored,
@@ -383,6 +384,6 @@ ext-import-graph =
 int-import-graph = 
 
 [EXCEPTIONS]
-overgeneral-exceptions = Exception
+overgeneral-exceptions = builtins.Exception
 
-# 6a5f2b58c0741088bb34597f419f7419e326a841
+# 6ba39b4accf3da730ae894307bd8cd4a4e122ab3
diff --git a/repo_health/check_pinned_python_dependencies.py b/repo_health/check_pinned_python_dependencies.py
@@ -0,0 +1,42 @@
+"""
+Counts the python dependencies which are pinned
+"""
+import os
+
+import pytest
+
+from repo_health import get_file_content
+
+module_dict_key = "pinned_python_dependencies"
+
+
+def get_dependencies_count(repo_path, file_name):
+    """
+   entry point to read requirements from constraints and common-constraints
+   @param repo_path:
+   @param file_name:
+   @return: number.
+   """
+    full_path = os.path.join(repo_path, "requirements/{0}".format(file_name))
+    content = get_file_content(full_path)
+    lines = content.split('\n')
+    dependency_count = 0
+    pinned_dependencies_marker = ['==', '>', '<']
+    for line in lines:
+        if line.startswith('#'):
+            continue
+        if any(marker in line for marker in pinned_dependencies_marker):
+            dependency_count += 1
+        else:
+            continue
+    return dependency_count
+
+
+@pytest.mark.edx_health
+def check_pinned_python_dependencies(repo_path, all_results):
+    """
+    We shall read constraints file
+    """
+    constraints_count = get_dependencies_count(repo_path, 'common_constraints.txt')
+    common_constraints_count = get_dependencies_count(repo_path, 'constraints.txt')
+    all_results[module_dict_key] = constraints_count + common_constraints_count
diff --git a/repo_health/check_renovate.py b/repo_health/check_renovate.py
@@ -143,6 +143,6 @@ async def check_renovate(all_results, repo_path, github_repo):
     all_results[MODULE_DICT_KEY] = {
         'configured': config_exists,
         'last_pr': await get_last_pull_date(github_repo) if config_exists else None,
-        'total_open_prs': total_open_prs if config_exists else None,
-        'oldest_open_pr_date': oldest_pr_date if config_exists else None,
+        'total_open_prs': total_open_prs if config_exists else None,  # pylint: disable=possibly-used-before-assignment
+        'oldest_open_pr_date': oldest_pr_date if config_exists else None,  # pylint: disable=possibly-used-before-assignment
     }
diff --git a/repo_health/check_ubuntufiles.py b/repo_health/check_ubuntufiles.py
@@ -232,7 +232,7 @@ def get_playbook_data(self, playbook_path):
             packages = self._prepare_data(list(packages))
             return packages
 
-        except Exception as exc:  # pylint: disable=useless-suppression
+        except Exception as exc:  # pylint: disable=broad-exception-caught
             logger.exception("Following error occurred while parsing yml playbook (%s) in configuration repo: %s",
                              playbook_path, exc)
             return []

diff --git a/repo_health/utils.py b/repo_health/utils.py
@@ -136,7 +136,7 @@ def get_release_tags(repo_dir):
             return all_tags_list[:all_tags_list.index(latest_tag) + 1]
         else:
             return None
-    except Exception as ex:
+    except Exception as ex:  # pylint: disable=broad-exception-caught
         print(str(ex))
         return None
 

diff --git a/requirements/base.txt b/requirements/base.txt
@@ -1,29 +1,27 @@
 #
-# This file is autogenerated by pip-compile with Python 3.8
+# This file is autogenerated by pip-compile with Python 3.11
 # by the following command:
 #
 #    make upgrade
 #
-aiohttp==3.9.3
+aiohappyeyeballs==2.4.0
+    # via aiohttp
+aiohttp==3.10.5
     # via
     #   github-py
     #   pytest-aiohttp
 aiosignal==1.3.1
     # via aiohttp
-async-timeout==4.0.3
-    # via aiohttp
-attrs==23.2.0
+attrs==24.2.0
     # via aiohttp
-cachetools==5.3.3
+cachetools==5.5.0
     # via google-auth
-certifi==2024.2.2
+certifi==2024.8.30
     # via requests
 charset-normalizer==3.3.2
     # via requests
 dockerfile==3.3.1
     # via -r requirements/base.in
-exceptiongroup==1.2.0
-    # via pytest
 frozenlist==1.4.1
     # via
     #   aiohttp
@@ -32,19 +30,19 @@ gitdb==4.0.11
     # via gitpython
 github-py @ git+https://github.com/ShineyDev/github.py.git@8e3514010eb35a91e012f2935f48138b3c35cef6
     # via -r requirements/github.in
-gitpython==3.1.42
+gitpython==3.1.43
     # via pytest-repo-health
-google-auth==2.28.1
+google-auth==2.34.0
     # via
     #   google-auth-oauthlib
     #   gspread
-google-auth-oauthlib==1.2.0
+google-auth-oauthlib==1.2.1
     # via gspread
 gspread==5.11.3
     # via
     #   -c requirements/constraints.txt
     #   -r requirements/base.in
-idna==3.6
+idna==3.8
     # via
     #   requests
     #   yarl
@@ -60,15 +58,15 @@ packaging==21.3
     # via
     #   -c requirements/constraints.txt
     #   pytest
-pluggy==1.4.0
+pluggy==1.5.0
     # via pytest
-pyasn1==0.5.1
+pyasn1==0.6.0
     # via
     #   pyasn1-modules
     #   rsa
-pyasn1-modules==0.3.0
+pyasn1-modules==0.4.0
     # via google-auth
-pyparsing==3.1.2
+pyparsing==3.1.4
     # via packaging
 pytest==8.0.2
     # via
@@ -78,29 +76,27 @@ pytest==8.0.2
     #   pytest-repo-health
 pytest-aiohttp==1.0.5
     # via pytest-repo-health
-pytest-asyncio==0.23.5
+pytest-asyncio==0.23.8
     # via
     #   -r requirements/base.in
     #   pytest-aiohttp
 pytest-repo-health==3.0.2
     # via -r requirements/base.in
-pyyaml==6.0.1
+pyyaml==6.0.2
     # via
     #   -r requirements/base.in
     #   pytest-repo-health
-requests==2.31.0
+requests==2.32.3
     # via requests-oauthlib
-requests-oauthlib==1.3.1
+requests-oauthlib==2.0.0
     # via google-auth-oauthlib
 rsa==4.9
     # via google-auth
 smmap==5.0.1
     # via gitdb
 toml==0.10.2
     # via -r requirements/base.in
-tomli==2.0.1
-    # via pytest
-urllib3==2.2.1
+urllib3==2.2.2
     # via requests
-yarl==1.9.4
+yarl==1.9.11
     # via aiohttp
diff --git a/requirements/ci.txt b/requirements/ci.txt
@@ -1,18 +1,18 @@
 #
-# This file is autogenerated by pip-compile with Python 3.8
+# This file is autogenerated by pip-compile with Python 3.11
 # by the following command:
 #
 #    make upgrade
 #
-cachetools==5.3.3
+cachetools==5.5.0
     # via tox
 chardet==5.2.0
     # via tox
 colorama==0.4.6
     # via tox
 distlib==0.3.8
     # via virtualenv
-filelock==3.13.1
+filelock==3.15.4
     # via
     #   tox
     #   virtualenv
@@ -21,21 +21,17 @@ packaging==21.3
     #   -c requirements/constraints.txt
     #   pyproject-api
     #   tox
-platformdirs==4.2.0
+platformdirs==4.2.2
     # via
     #   tox
     #   virtualenv
-pluggy==1.4.0
+pluggy==1.5.0
     # via tox
-pyparsing==3.1.2
+pyparsing==3.1.4
     # via packaging
 pyproject-api==1.5.0
     # via tox
-tomli==2.0.1
-    # via
-    #   pyproject-api
-    #   tox
 tox==4.0.0
     # via -r requirements/ci.in
-virtualenv==20.25.1
+virtualenv==20.26.3
     # via tox
diff --git a/requirements/common_constraints.txt b/requirements/common_constraints.txt
@@ -18,7 +18,15 @@ Django<5.0
 
 # elasticsearch>=7.14.0 includes breaking changes in it which caused issues in discovery upgrade process.
 # elastic search changelog: https://www.elastic.co/guide/en/enterprise-search/master/release-notes-7.14.0.html
+# See https://github.com/openedx/edx-platform/issues/35126 for more info
 elasticsearch<7.14.0
 
 # django-simple-history>3.0.0 adds indexing and causes a lot of migrations to be affected
 django-simple-history==3.0.0
+
+# Cause: https://github.com/openedx/event-tracking/pull/290
+# event-tracking 2.4.1 upgrades to pymongo 4.4.0 which is not supported on edx-platform.
+# We will pin event-tracking to do not break existing installations
+# This can be unpinned once https://github.com/openedx/edx-platform/issues/34586
+# has been resolved and edx-platform is running with pymongo>=4.4.0
+event-tracking<2.4.1
diff --git a/requirements/constraints.txt b/requirements/constraints.txt
@@ -15,13 +15,18 @@
 # greater version breaking test.
 packaging==21.3
 
-# gspread==5.12.0 contains breaking changes 
+# gspread==5.12.0 contains breaking changes
 # which break the scheduled repo health job
 gspread<5.12.0
 
 # pytest==8.1.0 breaks test plugin [needs to be investigated separately]
 pytest<8.1.0
 
-# Needed for Python 3.12 compatibility. 
+# Needed for Python 3.12 compatibility.
 # Can be removed once support for Python<3.12 is dropped.
 backports-zoneinfo==0.2.1; python_version < "3.9"
+
+# date added: 13-09-24
+# setuptools > 70.3.0 breaks the repo health workflow link: https://github.com/edx/repo-health-data/actions/runs/10846619044/job/30103106806
+# Issue to remove the pin https://github.com/openedx/edx-repo-health/issues/523
+setuptools==70.3.0