MAINT Removes GitPython dependency (flyteorg#1966)

* MAINT Removes GitPython dependency Signed-off-by: Thomas J. Fan <thomasjpfan@gmail.com> * MAINT Remove GitPython from requirements Signed-off-by: Thomas J. Fan <thomasjpfan@gmail.com> * STY Run linter Signed-off-by: Thomas J. Fan <thomasjpfan@gmail.com> * ENH Adds support for http Signed-off-by: Thomas J. Fan <thomasjpfan@gmail.com> --------- Signed-off-by: Thomas J. Fan <thomasjpfan@gmail.com> Signed-off-by: Rafael Raposo <rafaelraposo@spotify.com>
RRap0so · Dec 15, 2023 · 11aafbf · 11aafbf
1 parent ccc61cf
commit 11aafbf
Show file tree

Hide file tree

Showing 7 changed files with 74 additions and 28 deletions.
diff --git a/dev-requirements.txt b/dev-requirements.txt
@@ -145,10 +145,6 @@ gast==0.5.4
     # via tensorflow
 gcsfs==2023.9.2
     # via flytekit
-gitdb==4.0.11
-    # via gitpython
-gitpython==3.1.40
-    # via flytekit
 google-api-core[grpc]==2.12.0
     # via
     #   google-api-core
@@ -484,8 +480,6 @@ six==1.16.0
     #   kubernetes
     #   python-dateutil
     #   tensorflow
-smmap==5.0.1
-    # via gitdb
 sortedcontainers==2.4.0
     # via
     #   flytekit

diff --git a/doc-requirements.txt b/doc-requirements.txt
@@ -309,7 +309,6 @@ gitdb==4.0.10
     # via gitpython
 gitpython==3.1.36
     # via
-    #   flytekit
     #   mlflow
 google-api-core[grpc]==2.11.1
     # via

diff --git a/flytekit/remote/remote.py b/flytekit/remote/remote.py
@@ -6,6 +6,7 @@
 from __future__ import annotations
 
 import base64
+import configparser
 import functools
 import hashlib
 import os
@@ -147,14 +148,31 @@ def _get_git_repo_url(source_path):
     Get git repo URL from remote.origin.url
     """
     try:
-        from git import Repo
+        git_config = source_path / ".git" / "config"
+        if not git_config.exists():
+            raise ValueError(f"{source_path} is not a git repo")
+
+        config = configparser.ConfigParser()
+        config.read(git_config)
+        url = config['remote "origin"']["url"]
+
+        if url.startswith("git@"):
+            # url format: git@github.com:flytekit/flytekit.git
+            prefix_len, suffix_len = len("git@"), len(".git")
+            return url[prefix_len:-suffix_len].replace(":", "/")
+        elif url.startswith("https://"):
+            # url format: https://github.com/flytekit/flytekit
+            prefix_len = len("https://")
+            return url[prefix_len:]
+        elif url.startswith("http://"):
+            # url format: http://github.com/flytekit/flytekit
+            prefix_len = len("http://")
+            return url[prefix_len:]
+        else:
+            raise ValueError("Unable to parse url")
 
-        return "github.com/" + Repo(source_path).remotes.origin.url.split(".git")[0].split(":")[-1]
-    except ImportError:
-        remote_logger.warning("Could not import git. is the git executable installed?")
-    except Exception:
-        # If the file isn't in the git repo, we can't get the url from git config
-        remote_logger.debug(f"{source_path} is not a git repo.")
+    except Exception as e:
+        remote_logger.debug(str(e))
         return ""
 
 

diff --git a/plugins/flytekit-whylogs/requirements.txt b/plugins/flytekit-whylogs/requirements.txt
@@ -58,10 +58,6 @@ flyteidl==1.3.14
     # via flytekit
 flytekit==1.3.1
     # via flytekitplugins-whylogs
-gitdb==4.0.10
-    # via gitpython
-gitpython==3.1.31
-    # via flytekit
 googleapis-common-protos==1.59.0
     # via
     #   flyteidl
@@ -202,8 +198,6 @@ six==1.16.0
     # via
     #   asttokens
     #   python-dateutil
-smmap==5.0.0
-    # via gitdb
 sortedcontainers==2.4.0
     # via flytekit
 stack-data==0.6.2

diff --git a/setup.py b/setup.py
@@ -43,7 +43,6 @@
         "flyteidl>=1.10.0",
         "fsspec>=2023.3.0,<=2023.9.2",
         "gcsfs",
-        "gitpython",
         "googleapis-common-protos>=1.57",
         "grpcio",
         "grpcio-status",

diff --git a/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt b/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt
@@ -110,10 +110,6 @@ fsspec==2023.6.0
     #   s3fs
 gcsfs==2023.6.0
     # via flytekit
-gitdb==4.0.10
-    # via gitpython
-gitpython==3.1.37
-    # via flytekit
 google-api-core==2.11.1
     # via
     #   google-cloud-core
@@ -328,8 +324,6 @@ six==1.16.0
     #   isodate
     #   kubernetes
     #   python-dateutil
-smmap==5.0.0
-    # via gitdb
 sortedcontainers==2.4.0
     # via flytekit
 statsd==3.3.0

diff --git a/tests/flytekit/unit/remote/test_remote.py b/tests/flytekit/unit/remote/test_remote.py
@@ -1,5 +1,7 @@
 import os
 import pathlib
+import shutil
+import subprocess
 import tempfile
 import typing
 import uuid
@@ -28,7 +30,7 @@
 from flytekit.models.task import Task
 from flytekit.remote import FlyteTask
 from flytekit.remote.lazy_entity import LazyEntity
-from flytekit.remote.remote import FlyteRemote
+from flytekit.remote.remote import FlyteRemote, _get_git_repo_url
 from flytekit.tools.translator import Options, get_serializable, get_serializable_launch_plan
 from tests.flytekit.common.parameterizers import LIST_OF_TASK_CLOSURES
 
@@ -435,3 +437,49 @@ def test_execution_name(mock_client, mock_uuid):
             execution_name="execution-test",
             execution_name_prefix="execution-test",
         )
+
+
+@pytest.mark.parametrize(
+    "url, host",
+    [
+        ("https://github.com/flytekit/flytekit", "github.com"),
+        ("http://github.com/flytekit/flytekit", "github.com"),
+        ("git@github.com:flytekit/flytekit.git", "github.com"),
+        ("https://gitlab.com/flytekit/flytekit", "gitlab.com"),
+        ("git@gitlab.com:flytekit/flytekit.git", "gitlab.com"),
+    ],
+)
+def test_get_git_repo_url(url, host, tmp_path):
+    git_exec = shutil.which("git")
+    if git_exec is None:
+        pytest.skip("git is not installed")
+
+    source_path = tmp_path / "repo_source"
+    source_path.mkdir()
+    subprocess.check_output([git_exec, "init"], cwd=source_path)
+    subprocess.check_output([git_exec, "remote", "add", "origin", url], cwd=source_path)
+
+    returned_url = _get_git_repo_url(source_path)
+    assert returned_url == f"{host}/flytekit/flytekit"
+
+
+def test_get_git_report_url_not_a_git_repo(tmp_path):
+    """Check url when source path is not a git repo."""
+    source_path = tmp_path / "repo_source"
+    source_path.mkdir()
+    assert _get_git_repo_url(source_path) == ""
+
+
+def test_get_git_report_url_unknown_url(tmp_path):
+    """Check url when url is unknown."""
+    git_exec = shutil.which("git")
+    if git_exec is None:
+        pytest.skip("git is not installed")
+
+    source_path = tmp_path / "repo_source"
+    source_path.mkdir()
+    subprocess.check_output([git_exec, "init"], cwd=source_path)
+    subprocess.check_output([git_exec, "remote", "add", "origin", "unknown"], cwd=source_path)
+
+    returned_url = _get_git_repo_url(source_path)
+    assert returned_url == ""