chore: Update Python2To3Migrator to only migrate .py files

action.yml

@@ -2,10 +2,6 @@
 name: 'TED run'
 description: 'Run TED on a given repo'
 inputs:
-  git-repo:  # repo to work on
-    description: 'Git Repository'
-    required: true
-    default: ''
   git-branch:  # branch to work on
     description: 'Git Branch'
     required: false

services/Python2To3Migrator.py

@@ -61,7 +61,7 @@ def run_generation(self, retriever, llm, output_parser, clone_dir) -> None:
         answer = chain.invoke("Give me the list of files that should be modified to migrate from Python 2 to Python 3.")
 
         print("-------------------------------------------------\n")
-        print(answer)
+        print(f"🆗: {answer}")
         parsed = re.search('```json\n([\\w\\W]*?)\n```', answer)
 
         if parsed is not None:
@@ -84,14 +84,11 @@ def run_generation(self, retriever, llm, output_parser, clone_dir) -> None:
                 else:
                     print("File parsing failure")
         else:
-            print("Answer parsing failure")
+            print("🆘 Answer parsing failure")
 
     def get_file_extensions(self) -> List[str]:
-        return [".py", ".md", ".txt"]
-
-    def get_file_glob(self) -> str:
-        return "**/*"
-
+        return [".py"]
+
     def get_text_format(self) -> Language:
         return Language.PYTHON
 

services/TEDGenerator.py

@@ -12,10 +12,6 @@ def get_file_extensions(self) -> List[str]:
         """Returns the file extension repo code to take into account."""
         pass
 
-    def get_file_glob(self) -> str:
-        """Returns the glob to match files in repo code."""
-        pass
-
     def get_text_format(self) -> Language:
         """Returns the text format of the generator."""
         pass

services/UnitTestsGenerator.py

@@ -85,9 +85,6 @@ class  <className><suffix>Test {{
     def get_file_extensions(self) -> List[str]:
         return [".java"]
 
-    def get_file_glob(self) -> str:
-        return "**/*.java"
-
     def get_text_format(self) -> Language:
         return Language.JAVA
 

ted.py

@@ -2,6 +2,8 @@
 import argparse
 from uuid import uuid1
 from dotenv import load_dotenv
+from pprint import pprint
+from git import Repo
 
 from helpers.GitHelper import GitHelper
 from services.UnitTestsGenerator import UnitTestsGenerator
@@ -10,7 +12,10 @@
 from langchain_core.output_parsers import StrOutputParser
 from langchain_openai import AzureChatOpenAI, AzureOpenAIEmbeddings
 from langchain_community.document_loaders import GitLoader, DirectoryLoader, TextLoader
+from langchain_community.document_loaders.generic import GenericLoader
+from langchain_community.document_loaders.parsers import LanguageParser
 from langchain_text_splitters import RecursiveCharacterTextSplitter
+from langchain_community.document_loaders.parsers.txt import TextParser
 
 def main():
 
@@ -50,47 +55,77 @@ def main():
     )
 
     output_parser = StrOutputParser()
-
+    loaders = []
     # Check if git_url exists
     clone_path = None
     if git_url:
         print(f"Loaded clones repository from URL: {git_url}")
         clone_path="./clone/"
-        loader = GitLoader(
-            clone_url=git_url,
-            repo_path=clone_path,
-            branch=branch,
-            file_filter=lambda file_path: filter_files(file_path, generator.get_file_extensions())
-        )
-    else:
+
+        if(os.path.exists(clone_path)):
+            print("Repository already cloned. Skip cloning.")
+        else:
+            print(f"Clone repository from {git_url} to {clone_path}")
+            Repo.clone_from(
+                url=git_url,
+                single_branch=True,
+                depth=1,
+                to_path=clone_path,
+                branch=branch,
+            )
+
+    if(not clone_path and os.getenv('GITHUB_WORKSPACE')):
         clone_path=os.getenv('GITHUB_WORKSPACE')
         print(f"Loader uses from github workspace: {clone_path}")
-        loader = DirectoryLoader(
-            path=clone_path,
-            glob=generator.get_file_glob(), # @TODO Find a way to use glob with extensions: "**/*{" +",".join(generator.getFileExtensions()) + "}",
-            exclude=["**/*.yml", "**/*.json"],
-            show_progress=True,
-            loader_cls=TextLoader
-        )
-
-    text_format = generator.get_text_format()
 
-    print("Load documents")
-    docs = loader.load()
+    if(not clone_path):
+        print("No git repository provided.")
+        return
+
+    print(f'📂 Load documents from {clone_path}')
+    loaders.append(GenericLoader.from_filesystem(
+        clone_path,
+        glob="*",
+        #suffixes= generator.get_file_extensions(),
+        #parser=LanguageParser(generator.get_text_format()),
+        suffixes= [".py"],
+        parser=LanguageParser(generator.get_text_format()),
+    ))
+    loaders.append(GenericLoader.from_filesystem(
+        clone_path,
+        glob="*",
+        suffixes= [".txt", ".md", "Dockerfile"],
+        parser= TextParser()
+    ))
+
+    # Load all documents
+    docs = []
+    for loader in loaders:
+        docs.extend(loader.load())
+
+    # Add a new document to docs array
+    # docs.append(TextLoader("text", "This is a test document"))
 
     # if zero docs stop
     if len(docs) == 0:
-        print("No documents found.")
+        print("📄 No documents found.")
         return
-
+    print(f"📄 Found {len(docs)} documents")
+
+    for document in docs:
+        pprint(document)
+
+
+    text_format = generator.get_text_format()
     print("Using language splitter {}.".format(text_format))
     text_splitter = RecursiveCharacterTextSplitter.from_language(
         language=text_format ,chunk_size=2000, chunk_overlap=200
     )
 
     print("Split documents")
     texts = text_splitter.split_documents(docs)
-
+
+    print(f"📄 Generated {len(texts)} chuncks for {len(docs)} documents")
     print("Create embeddings and vector store")
     embedding = AzureOpenAIEmbeddings(
         # keys and endpoint are read from the .env file
@@ -102,7 +137,7 @@ def main():
     )
     retriever = vector_store.as_retriever()
 
-    print("😀 Run generation")
+    print("🔍 Run generation")
     generator.run_generation(retriever, llm, output_parser, clone_path)
 
     if(push and github_repository and branch and github_token):

tests/services/test_Generator.py

@@ -13,10 +13,6 @@ def test_get_file_extensions(self):
         ted_generator = TEDGenerator()
         self.assertIsNone(ted_generator.get_file_extensions())
 
-    def test_get_file_glob(self):
-        ted_generator = TEDGenerator()
-        self.assertIsNone(ted_generator.get_file_glob())
-
     def test_get_text_format(self):
         ted_generator = TEDGenerator()
         self.assertIsNone(ted_generator.get_text_format())