further refactors + custom scheduler

.gitignore

@@ -1,4 +1,5 @@
 llama.cpp/*
 logs/*
 .vscode/*
-TODO.md
+TODO.md
+testing/*

Homepage.py

@@ -10,9 +10,11 @@
     Page("pages/Docs.py", "Docs", ":books:"),
     Section("Manually convert models", icon=":arrows_counterclockwise:"),
     Page("pages/Hugging_Face_Downloader.py", "Download model", ":inbox_tray:"),
-    Page("pages/High_Precision_Quantization.py", "High Precision Quantization", ":gem:"),
-    Page("pages/Medium_Precision_Quantization.py", "Medium Precision Quantization", ":heavy_plus_sign:" ),
+    Page("pages/Convert_Safetensors.py", "Convert Safetensors to High Precision", ":gem:"),
+    Page("pages/Create_IMatrix.py", "Create Importance Matrix", ":chart_with_upwards_trend:"),
+    Page("pages/Quantize_GGUF.py", "Quantize GGUF", ":heavy_plus_sign:" ),
     Page("pages/Upload_Converted_To_HF.py", "Upload model to HuggingFace", ":outbox_tray:"),
+    Page("pages/Queue_GUI.py", "Queue GUI", ":inbox_tray:"),
     Section("Extra Tools", icon=":toolbox:"),
     Page("pages/HF_Token_Encrypter.py", "Security", ":lock:"),
     ]    

pages/High_Precision_Quantization.py → pages/Convert_Safetensors.py

@@ -1,21 +1,15 @@
 # IMPORTS ---------------------------------------------------------------------------------
+import subprocess, threading, queue, streamlit as st
 from apscheduler.schedulers.background import BackgroundScheduler
-import subprocess
-import threading
-from pathlib import Path
-import streamlit as st
-from modules.shared import shared
-import sys
-import queue
-from tools.find import find_llama_models_dir
+from st_pages import add_indentation
+from util.constants import config
+from util.find import find_llama_models_dir
 
 # FUNCTIONS ---------------------------------------------------------------------------------
 
 # Initialize queue
 command_queue = queue.Queue()
 
-# Existing function definitions (find_llama_models_dir, run_command, trigger_command)
-
 def process_queue():
     if not command_queue.empty():
         model_folder, out_type = command_queue.get()
@@ -65,13 +59,15 @@ def trigger_command(model_folder, options):
 
 # UI CODE ---------------------------------------------------------------------------------
 
+add_indentation()
+
 st.title("High Precision Quantization")
 
 models_dir = Path("llama.cpp/models/")
 model_folders = [f.name for f in models_dir.iterdir() if f.is_dir()] if models_dir.exists() else ["Directory not found"]
 
 model_folder = st.selectbox("Select a Model Folder", model_folders)
-options = {option: st.checkbox(label=option) for option in shared['checkbox_high_options']}
+options = {option: st.checkbox(label=option) for option in config['checkbox_high_options']}
 
 if st.button("Run Commands"):
     if not any(options.values()):

pages/Create_IMatrix.py

@@ -0,0 +1,18 @@
+# IMPORTS ---------------------------------------------------------------------------------
+import streamlit as st
+from st_pages import add_indentation
+
+# FUNCTIONS ---------------------------------------------------------------------------------
+
+# UI CODE ---------------------------------------------------------------------------------
+
+add_indentation()
+
+st.title("Create Importance Matrix")
+
+# TODO make the entire  damn page lmao
+with st.expander("What is an Importance Matrix?"):
+    st.markdown("""
+    An Importance Matrix is a tool used to determine the importance of each feature in a dataset. 
+    It is a matrix that shows how much each feature contributes to the prediction of the target variable.
+    """)

pages/Docs.py

@@ -1,11 +1,12 @@
+# IMPORTS ---------------------------------------------------------------------------------
 import streamlit as st
-from modules.docs_inline import docs
-from st_pages import Page,  add_indentation
-
-add_indentation()
+from util.docs_inline import docs
+from st_pages import add_indentation
 
 # UI CODE ---------------------------------------------------------------------------------
 
+add_indentation()
+
 # Create Tabs for Main Subjects
 tab_main = st.tabs(list(docs.keys()))
 

pages/HF_Token_Encrypter.py

@@ -1,48 +1,12 @@
 # IMPORTS ---------------------------------------------------------------------------------
 import streamlit as st
-from cryptography.fernet import Fernet
-from pathlib import Path
-import subprocess
-
-# FUNCTIONS ---------------------------------------------------------------------------------
-
-# Function to load the existing key
-def load_key():
-    key_dir = Path('.') / '.key'
-    key_file_path = key_dir / 'encryption.key'
-    return key_file_path.read_bytes()
-
-# Encrypt the token
-def encrypt_token(token):
-    key = load_key()
-    f = Fernet(key)
-    encrypted_token = f.encrypt(token.encode())
-    return encrypted_token.decode()
-
-def generate_new_key():
-    key_dir = Path('.') / '.key'
-    key_file_path = key_dir / 'encryption.key'
-
-    # Check if the key file exists and delete it
-    if key_file_path.exists():
-        try:
-            key_file_path.unlink()  # Deletes the file
-            print("Existing key file deleted.")
-        except Exception as e:
-            return f"Error deleting existing key: {e}"
-
-    # Generate new key
-    root_dir = Path(__file__).parent.parent
-    script_path = root_dir / 'tools/key_generation.py'
-    try:
-        subprocess.run(['python3', str(script_path)], check=True)
-        return "New private key generated successfully."
-    except subprocess.CalledProcessError as e:
-        return f"Error generating new key: {e}"
-
+from st_pages import add_indentation
+from util.key import encrypt_token, generate_new_key
 
 # UI CODE ---------------------------------------------------------------------------------
 
+add_indentation()
+
 # def show_token_encrypt_page():
 st.title("Token Encryption")
 

pages/Hugging_Face_Downloader.py

@@ -1,10 +1,8 @@
 # IMPORTS ---------------------------------------------------------------------------------
-import os
-import subprocess
-import streamlit as st
-import requests
+import os, subprocess, requests, streamlit as st
 from apscheduler.schedulers.background import BackgroundScheduler
 from pathlib import Path
+from st_pages import add_indentation
 
 # FUNCTIONS ---------------------------------------------------------------------------------
 
@@ -87,6 +85,8 @@ def get_files_from_repo(url, repo_name):
 
 # UI CODE ---------------------------------------------------------------------------------
 
+add_indentation()
+
 st.title("Model Downloader")
 
 model_name = st.text_input("Download PyTorch models from Huggingface", "Use the HuggingfaceUsername/Modelname")

pages/Medium_Precision_Quantization.py → pages/Quantize_GGUF.py

@@ -1,36 +1,17 @@
 # IMPORTS ---------------------------------------------------------------------------------
-import os
-import subprocess
-import streamlit as st
-from modules.shared import shared
+import os, subprocess, sys, streamlit as st
 from apscheduler.schedulers.background import BackgroundScheduler
 from pathlib import Path
-import sys
+from st_pages import add_indentation
+from util.constants import config
+from util.find import find_llama_cpp_dir
 
 # FUNCTIONS ---------------------------------------------------------------------------------
 
 # Initialize the scheduler
 scheduler = BackgroundScheduler()
 scheduler.start()
 
-def find_llama_cpp_dir():
-    # Search for llama.cpp directory two levels up
-    current_dir = Path(__file__).resolve().parent
-    for _ in range(2):
-        current_dir = current_dir.parent
-        llama_cpp_dir = current_dir / 'llama.cpp'
-        if llama_cpp_dir.is_dir():
-            return llama_cpp_dir
-
-    # If not found, search two levels down
-    current_dir = Path(__file__).resolve().parent
-    for _ in range(2):
-        current_dir = current_dir / 'llama.cpp'
-        if current_dir.is_dir():
-            return current_dir
-
-    return None
-
 def list_gguf_files(models_dir):
     gguf_files = []
     if os.path.exists(models_dir):
@@ -109,6 +90,8 @@ def trigger_command(modelpath, options, use_docker):
 
 # UI CODE ---------------------------------------------------------------------------------
 
+add_indentation()
+
 st.title("Medium Precision Quantization")
 
 models_dir = os.path.join("llama.cpp", "models")