From 551eed76d2ec5207b609d1cd9861a7115b0f3a9c Mon Sep 17 00:00:00 2001 From: bmaltais Date: Wed, 21 Jun 2023 15:09:18 -0400 Subject: [PATCH] Refactor setup (#1030) * Refactor setup in setup folder * More refactoring * Remove need to run setup.sh as sudo * Fix for unix undel python 3.8 * Create setup_common.py * Fix windows setup * Refining setup --------- Co-authored-by: Your Name --- .gitignore | 1 + Dockerfile | 4 +- README.md | 2 + gui.bat | 4 +- gui.ps1 | 4 +- gui.sh | 10 +- library/custom_logging.py | 6 +- pyproject.toml | 24 ++ ...Ubuntu_20.04.txt => requirements_linux.txt | 11 +- ...rements_unix.txt => requirements_macos.txt | 10 +- setup.bat | 4 +- setup.ps1 | 4 +- setup.py | 10 - setup.sh | 162 +++++++------ {tools => setup}/check_local_modules.py | 0 {tools => setup}/create_user_files.py | 0 {tools => setup}/debug_info.py | 0 .../setup_windows.py => setup/setup_common.py | 226 ++---------------- setup/setup_windows.py | 194 +++++++++++++++ {tools => setup}/update_bitsandbytes.py | 0 .../validate_requirements.py | 25 +- tools/cudann_1.8_install.py | 26 -- tools/validate_requirements.py | 122 ---------- upgrade.bat | 2 +- upgrade.sh | 16 -- 25 files changed, 378 insertions(+), 489 deletions(-) create mode 100644 pyproject.toml rename requirements_Ubuntu_20.04.txt => requirements_linux.txt (57%) rename requirements_unix.txt => requirements_macos.txt (56%) delete mode 100644 setup.py rename {tools => setup}/check_local_modules.py (100%) rename {tools => setup}/create_user_files.py (100%) rename {tools => setup}/debug_info.py (100%) rename tools/setup_windows.py => setup/setup_common.py (69%) create mode 100644 setup/setup_windows.py rename {tools => setup}/update_bitsandbytes.py (100%) rename tools/validate_requirements_unix.py => setup/validate_requirements.py (84%) delete mode 100644 tools/cudann_1.8_install.py delete mode 100644 tools/validate_requirements.py delete mode 100644 upgrade.sh diff --git a/.gitignore b/.gitignore index ccb515218..e23600be3 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,4 @@ test/output test/logs test/*.json test/ft +requirements_tmp_for_setup.txt diff --git a/Dockerfile b/Dockerfile index a1377f7db..cfa0b5df9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,8 +26,8 @@ RUN python3 -m pip install wheel ## RUN python3 -m pip install -v -U git+https://github.com/facebookresearch/xformers.git@main#egg=xformers # Install requirements -COPY requirements_unix.txt setup.py ./ -RUN python3 -m pip install --use-pep517 -r requirements_unix.txt xformers +COPY requirements_linux.txt ./setup/setup.py ./ +RUN python3 -m pip install --use-pep517 -r requirements_linux.txt xformers # Replace pillow with pillow-simd RUN python3 -m pip uninstall -y pillow && \ diff --git a/README.md b/README.md index ef225b284..a527d1ab7 100644 --- a/README.md +++ b/README.md @@ -102,6 +102,8 @@ If you run on Linux, there is an alternative docker container port with less lim venv support need to be pre-installed. Can be done on ubuntu 22.04 with `apt install python3.10-venv` +For Linux, make sure to install the cudaNN drivers following the instructions from: `https://developer.nvidia.com/cuda-downloads?target_os=Linux&target_arch=x86_64` + Make sure to use a version of python >= 3.10.6 and < 3.11.0 #### Setup diff --git a/gui.bat b/gui.bat index 23768b525..15e360fe3 100644 --- a/gui.bat +++ b/gui.bat @@ -3,14 +3,14 @@ call .\venv\Scripts\deactivate.bat :: Calling external python program to check for local modules -python .\tools\check_local_modules.py --no_question +python .\setup\check_local_modules.py --no_question :: Activate the virtual environment call .\venv\Scripts\activate.bat set PATH=%PATH%;%~dp0venv\Lib\site-packages\torch\lib :: Validate requirements -python.exe .\tools\validate_requirements.py +python.exe .\setup\validate_requirements.py :: If the exit code is 0, run the kohya_gui.py script with the command-line arguments if %errorlevel% equ 0 ( diff --git a/gui.ps1 b/gui.ps1 index 8044e9304..2947343db 100644 --- a/gui.ps1 +++ b/gui.ps1 @@ -29,10 +29,10 @@ if ($pipOutput) { $env:PATH += ";$($MyInvocation.MyCommand.Path)\venv\Lib\site-packages\torch\lib" # Debug info about system -# python.exe .\tools\debug_info.py +# python.exe .\setup\debug_info.py # Validate the requirements and store the exit code -python.exe .\tools\validate_requirements.py +python.exe .\setup\validate_requirements.py # If the exit code is 0, read arguments from gui_parameters.txt (if it exists) # and run the kohya_gui.py script with the command-line arguments diff --git a/gui.sh b/gui.sh index 03ed5b00b..78ad7b4ee 100755 --- a/gui.sh +++ b/gui.sh @@ -17,6 +17,12 @@ cd "$SCRIPT_DIR" source "$SCRIPT_DIR/venv/bin/activate" # If the requirements are validated, run the kohya_gui.py script with the command-line arguments -if python "$SCRIPT_DIR"/tools/validate_requirements_unix.py -r "$SCRIPT_DIR"/requirements_unix.txt; then - python "$SCRIPT_DIR/kohya_gui.py" "$@" +if [[ "$OSTYPE" == "darwin"* ]]; then + if python "$SCRIPT_DIR"/setup/validate_requirements.py -r "$SCRIPT_DIR"/requirements_macos.txt; then + python "$SCRIPT_DIR/kohya_gui.py" "$@" + fi +else + if python "$SCRIPT_DIR"/setup/validate_requirements.py -r "$SCRIPT_DIR"/requirements_linux.txt; then + python "$SCRIPT_DIR/kohya_gui.py" "$@" + fi fi diff --git a/library/custom_logging.py b/library/custom_logging.py index f50aa1801..ee7e5e208 100644 --- a/library/custom_logging.py +++ b/library/custom_logging.py @@ -1,6 +1,7 @@ import os import logging import time +import sys from rich.theme import Theme from rich.logging import RichHandler @@ -23,7 +24,10 @@ def setup_logging(clean=False, debug=False): except: pass - logging.basicConfig(level=logging.DEBUG, format='%(asctime)s | %(levelname)s | %(pathname)s | %(message)s', filename='setup.log', filemode='a', encoding='utf-8', force=True) + if sys.version_info >= (3, 9): + logging.basicConfig(level=logging.DEBUG, format='%(asctime)s | %(levelname)s | %(pathname)s | %(message)s', filename='setup.log', filemode='a', encoding='utf-8', force=True) + else: + logging.basicConfig(level=logging.DEBUG, format='%(asctime)s | %(levelname)s | %(pathname)s | %(message)s', filename='setup.log', filemode='a', force=True) console = Console(log_time=True, log_time_format='%H:%M:%S-%f', theme=Theme({ "traceback.border": "black", diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..35f2ac908 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,24 @@ +[build-system] +requires = ["setuptools", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "kohya_ss" +version = "1.0.3" +description = "A GUI wrapper for kohya-ss SD scipts enabling LoRA training with an easy-to-use web application." +authors = [ + {name = "bmaltais", email = "bernard@ducourier.com"}, +] +readme = "README.md" +classifiers = [ + "Development Status :: 5 - Production/Stable", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", +] + +[tool.poetry] +license = "Apache-2.0" + +[tool.setuptools.packages.find] +where = ["library"] # We have to explicitly tell build tools where to look \ No newline at end of file diff --git a/requirements_Ubuntu_20.04.txt b/requirements_linux.txt similarity index 57% rename from requirements_Ubuntu_20.04.txt rename to requirements_linux.txt index dd5fc461e..eb0410452 100644 --- a/requirements_Ubuntu_20.04.txt +++ b/requirements_linux.txt @@ -8,10 +8,8 @@ easygui==0.98.3 einops==0.6.0 fairscale==0.4.13 ftfy==6.1.1 -gradio==3.23.0; sys_platform == 'darwin' -gradio==3.32.0; sys_platform != 'darwin' -huggingface-hub==0.13.3; sys_platform == 'darwin' -huggingface-hub==0.13.3; sys_platform != 'darwin' +gradio==3.32.0 +huggingface-hub==0.13.3 lion-pytorch==0.0.6 lycoris_lora==0.1.6 opencv-python==4.7.0.68 @@ -19,9 +17,8 @@ prodigyopt==1.0 pytorch-lightning==1.9.0 rich==13.4.1 safetensors==0.2.6 -tensorboard==2.10.1 ; sys_platform != 'darwin' -tensorboard==2.12.1 ; sys_platform == 'darwin' -tensorflow==2.10.1; sys_platform != 'darwin' +tensorboard==2.12.1 +tensorflow==2.12.0 timm==0.6.12 tk==0.1.0 toml==0.10.2 diff --git a/requirements_unix.txt b/requirements_macos.txt similarity index 56% rename from requirements_unix.txt rename to requirements_macos.txt index fd10892ab..5fe1e87ed 100644 --- a/requirements_unix.txt +++ b/requirements_macos.txt @@ -8,19 +8,15 @@ easygui==0.98.3 einops==0.6.0 fairscale==0.4.13 ftfy==6.1.1 -gradio==3.23.0; sys_platform == 'darwin' -gradio==3.32.0; sys_platform != 'darwin' -huggingface-hub==0.13.0; sys_platform == 'darwin' -huggingface-hub==0.13.3; sys_platform != 'darwin' +gradio==3.23.0 +huggingface-hub==0.13.0 lion-pytorch==0.0.6 lycoris_lora==0.1.6 opencv-python==4.7.0.68 pytorch-lightning==1.9.0 rich==13.4.1 safetensors==0.2.6 -tensorboard==2.10.1 ; sys_platform != 'darwin' -tensorboard==2.12.1 ; sys_platform == 'darwin' -tensorflow==2.10.1; sys_platform != 'darwin' +tensorboard==2.12.1 timm==0.6.12 tk==0.1.0 toml==0.10.2 diff --git a/setup.bat b/setup.bat index 95d069e59..222e8033a 100644 --- a/setup.bat +++ b/setup.bat @@ -20,11 +20,11 @@ mkdir ".\logs\setup" > nul 2>&1 call .\venv\Scripts\deactivate.bat :: Calling external python program to check for local modules -python .\tools\check_local_modules.py +python .\setup\check_local_modules.py call .\venv\Scripts\activate.bat -python .\tools\setup_windows.py +python .\setup\setup_windows.py :: Deactivate the virtual environment call .\venv\Scripts\deactivate.bat \ No newline at end of file diff --git a/setup.ps1 b/setup.ps1 index 71c5d9245..df5f92e68 100644 --- a/setup.ps1 +++ b/setup.ps1 @@ -17,11 +17,11 @@ $null = New-Item -ItemType Directory -Force -Path ".\logs\setup" & .\venv\Scripts\deactivate.bat # Calling external python program to check for local modules -& .\venv\Scripts\python.exe .\tools\check_local_modules.py +& .\venv\Scripts\python.exe .\setup\check_local_modules.py & .\venv\Scripts\activate.bat -& .\venv\Scripts\python.exe .\tools\setup_windows.py +& .\venv\Scripts\python.exe .\setup\setup_windows.py # Deactivate the virtual environment & .\venv\Scripts\deactivate.bat diff --git a/setup.py b/setup.py deleted file mode 100644 index 8aa7c075d..000000000 --- a/setup.py +++ /dev/null @@ -1,10 +0,0 @@ -from setuptools import setup, find_packages -import subprocess -import os -import sys - -# Call the create_user_files.py script -script_path = os.path.join("tools", "create_user_files.py") -subprocess.run([sys.executable, script_path]) - -setup(name="library", version="1.0.3", packages=find_packages()) diff --git a/setup.sh b/setup.sh index 7f1a850c8..a95939959 100755 --- a/setup.sh +++ b/setup.sh @@ -3,8 +3,6 @@ # This file will be the host environment setup file for all operating systems other than base Windows. # Set the required package versions here. -# They will be appended to the requirements_unix.txt file in the installation directory. -TENSORFLOW_VERSION="2.12.0" TENSORFLOW_MACOS_VERSION="2.12.0" TENSORFLOW_METAL_VERSION="0.8.0" @@ -90,7 +88,7 @@ GIT_REPO="https://github.com/bmaltais/kohya_ss.git" INTERACTIVE=false PUBLIC=false SKIP_SPACE_CHECK=false -SKIP_GIT_UPDATE=false +SKIP_GIT_UPDATE=true SKIP_GUI=false while getopts ":vb:d:g:inprus-:" opt; do @@ -100,6 +98,7 @@ while getopts ":vb:d:g:inprus-:" opt; do OPTARG="${OPTARG#$opt}" # extract long option argument (may be empty) OPTARG="${OPTARG#=}" # if long option argument, remove assigning `=` fi + case $opt in b | branch) BRANCH="$OPTARG" ;; d | dir) DIR="$OPTARG" ;; @@ -194,28 +193,34 @@ size_available() { # The expected usage is create_symlinks symlink target_file create_symlinks() { + local symlink="$1" + local target_file="$2" + echo "Checking symlinks now." - # Next line checks for valid symlink - if [ -L "$1" ]; then + + # Check if the symlink exists + if [ -L "$symlink" ]; then # Check if the linked file exists and points to the expected file - if [ -e "$1" ] && [ "$(readlink "$1")" == "$2" ]; then - echo "$(basename "$1") symlink looks fine. Skipping." + if [ -e "$symlink" ] && [ "$(readlink "$symlink")" == "$target_file" ]; then + echo "$(basename "$symlink") symlink looks fine. Skipping." else - if [ -f "$2" ]; then - echo "Broken symlink detected. Recreating $(basename "$1")." - rm "$1" && - ln -s "$2" "$1" + if [ -f "$target_file" ]; then + echo "Broken symlink detected. Recreating $(basename "$symlink")." + rm "$symlink" && ln -s "$target_file" "$symlink" else - echo "$2 does not exist. Nothing to link." + echo "$target_file does not exist. Nothing to link." fi fi else - echo "Linking $(basename "$1")." - ln -s "$2" "$1" + echo "Linking $(basename "$symlink")." + ln -s "$target_file" "$symlink" fi } + install_python_dependencies() { + local TEMP_REQUIREMENTS_FILE + # Switch to local virtual env echo "Switching to virtual Python environment." if ! inDocker; then @@ -235,61 +240,55 @@ install_python_dependencies() { # Updating pip if there is one echo "Checking for pip updates before Python operations." - pip install --upgrade pip >&3 + pip install --upgrade pip echo "Installing python dependencies. This could take a few minutes as it downloads files." echo "If this operation ever runs too long, you can rerun this script in verbose mode to check." + case "$OSTYPE" in - "linux-gnu"*) pip install torch==2.0.1+cu118 torchvision==0.15.2+cu118 \ - --extra-index-url https://download.pytorch.org/whl/cu118 >&3 && - pip install -U -I xformers==0.0.20 >&3 ;; - "darwin"*) pip install torch==2.0.0 torchvision==0.15.1 \ - -f https://download.pytorch.org/whl/cpu/torch_stable.html >&3 ;; - "cygwin") - : - ;; - "msys") - : - ;; + "linux-gnu"*) + pip install torch==2.0.1+cu118 torchvision==0.15.2+cu118 \ + --extra-index-url https://download.pytorch.org/whl/cu118 + pip install --upgrade xformers==0.0.20 + ;; + "darwin"*) + pip install torch==2.0.0 torchvision==0.15.1 \ + -f https://download.pytorch.org/whl/cpu/torch_stable.html + # Check if the processor is Apple Silicon (arm64) + if [[ "$(uname -m)" == "arm64" ]]; then + pip install tensorflow-metal=="$TENSORFLOW_MACOS_VERSION" + else + pip install tensorflow-macos=="$TENSORFLOW_METAL_VERSION" + fi + ;; esac if [ "$RUNPOD" = true ]; then echo "Installing tenssort." - pip install tensorrt >&3 + pip install tensorrt fi # DEBUG ONLY (Update this version number to whatever PyCharm recommends) # pip install pydevd-pycharm~=223.8836.43 - #This will copy our requirements_unix.txt file out and make the khoya_ss lib a dynamic location then cleanup. - local TEMP_REQUIREMENTS_FILE="$DIR/requirements_tmp_for_setup.txt" - echo "Copying $DIR/requirements_unix.txt to $TEMP_REQUIREMENTS_FILE" >&3 - echo "Replacing the . for lib to our DIR variable in $TEMP_REQUIREMENTS_FILE." >&3 - awk -v dir="$DIR" '/#.*kohya_ss.*library/{print; getline; sub(/^\.$/, dir)}1' "$DIR/requirements_unix.txt" >"$TEMP_REQUIREMENTS_FILE" - - # This will check if macOS is running then determine if M1+ or Intel CPU. - # It will append the appropriate packages to the requirements_unix.txt file. - # Other OSs won't be affected and the version variables are at the top of this file. - if [[ "$(uname)" == "Darwin" ]]; then - # Check if the processor is Apple Silicon (arm64) - if [[ "$(uname -m)" == "arm64" ]]; then - echo "tensorflow-macos==$TENSORFLOW_MACOS_VERSION" >>"$TEMP_REQUIREMENTS_FILE" - echo "tensorflow-metal==$TENSORFLOW_METAL_VERSION" >>"$TEMP_REQUIREMENTS_FILE" - # Check if the processor is Intel (x86_64) - elif [[ "$(uname -m)" == "x86_64" ]]; then - echo "tensorflow==$TENSORFLOW_VERSION" >>"$TEMP_REQUIREMENTS_FILE" - fi - fi + # Create a temporary requirements file + TEMP_REQUIREMENTS_FILE=$(mktemp) - if [ $VERBOSITY == 2 ]; then - python -m pip install --quiet --use-pep517 --upgrade -r "$TEMP_REQUIREMENTS_FILE" >&3 + if [[ "$OSTYPE" == "darwin"* ]]; then + echo "Copying $DIR/requirements_macos.txt to $TEMP_REQUIREMENTS_FILE" >&3 + echo "Replacing the . for lib to our DIR variable in $TEMP_REQUIREMENTS_FILE." >&3 + awk -v dir="$DIR" '/#.*kohya_ss.*library/{print; getline; sub(/^\.$/, dir)}1' "$DIR/requirements_macos.txt" >"$TEMP_REQUIREMENTS_FILE" else - python -m pip install --use-pep517 --upgrade -r "$TEMP_REQUIREMENTS_FILE" >&3 + echo "Copying $DIR/requirements_linux.txt to $TEMP_REQUIREMENTS_FILE" >&3 + echo "Replacing the . for lib to our DIR variable in $TEMP_REQUIREMENTS_FILE." >&3 + awk -v dir="$DIR" '/#.*kohya_ss.*library/{print; getline; sub(/^\.$/, dir)}1' "$DIR/requirements_linux.txt" >"$TEMP_REQUIREMENTS_FILE" fi - echo "Removing the temp requirements file." - if [ -f "$TEMP_REQUIREMENTS_FILE" ]; then - rm -f "$TEMP_REQUIREMENTS_FILE" + # Install the Python dependencies from the temporary requirements file + if [ $VERBOSITY == 2 ]; then + python -m pip install --quiet --upgrade -r "$TEMP_REQUIREMENTS_FILE" + else + python -m pip install --upgrade -r "$TEMP_REQUIREMENTS_FILE" fi if [ -n "$VIRTUAL_ENV" ] && ! inDocker; then @@ -302,6 +301,7 @@ install_python_dependencies() { fi } + # Attempt to non-interactively install a default accelerate config file unless specified otherwise. # Documentation for order of precedence locations for configuration file for automated installation: # https://huggingface.co/docs/accelerate/basic_tutorials/launch#custom-configurations @@ -482,48 +482,62 @@ if [[ "$OSTYPE" == "linux-gnu"* ]]; then echo "Raw detected distro string: $distro" >&4 echo "Raw detected distro family string: $family" >&4 - echo "Installing Python TK if not found on the system." if "$distro" | grep -qi "Ubuntu" || "$family" | grep -qi "Ubuntu"; then echo "Ubuntu detected." if [ $(dpkg-query -W -f='${Status}' python3-tk 2>/dev/null | grep -c "ok installed") = 0 ]; then - if [ "$root" = true ]; then - apt update -y >&3 && apt install -y python3-tk >&3 - else - echo "This script needs to be run as root or via sudo to install packages." + # if [ "$root" = true ]; then + echo "This script needs you to install the missing python3-tk packages. Please install with:" + echo " " + echo "sudo apt update -y && sudo apt install -y python3-tk" exit 1 - fi + # else + # echo "This script needs to be run as root or via sudo to install packages." + # exit 1 + # fi else - echo "Python TK found! Skipping install!" + echo "Python TK found..." fi elif "$distro" | grep -Eqi "Fedora|CentOS|Redhat"; then echo "Redhat or Redhat base detected." if ! rpm -qa | grep -qi python3-tkinter; then - if [ "$root" = true ]; then - dnf install python3-tkinter -y >&3 - else - echo "This script needs to be run as root or via sudo to install packages." + # if [ "$root" = true ]; then + echo "This script needs you to install the missing python3-tk packages. Please install with:\n\n" + echo "sudo dnf install python3-tkinter -y >&3" exit 1 - fi + # else + # echo "This script needs to be run as root or via sudo to install packages." + # exit 1 + # fi + else + echo "Python TK found..." fi elif "$distro" | grep -Eqi "arch" || "$family" | grep -qi "arch"; then echo "Arch Linux or Arch base detected." if ! pacman -Qi tk >/dev/null; then - if [ "$root" = true ]; then - pacman --noconfirm -S tk >&3 - else - echo "This script needs to be run as root or via sudo to install packages." + # if [ "$root" = true ]; then + echo "This script needs you to install the missing python3-tk packages. Please install with:\n\n" + echo "pacman --noconfirm -S tk >&3" exit 1 - fi + # else + # echo "This script needs to be run as root or via sudo to install packages." + # exit 1 + # fi + else + echo "Python TK found..." fi elif "$distro" | grep -Eqi "opensuse" || "$family" | grep -qi "opensuse"; then echo "OpenSUSE detected." if ! rpm -qa | grep -qi python-tk; then - if [ "$root" = true ]; then - zypper install -y python-tk >&3 - else - echo "This script needs to be run as root or via sudo to install packages." + # if [ "$root" = true ]; then + echo "This script needs you to install the missing python3-tk packages. Please install with:\n\n" + echo "zypper install -y python-tk >&3" exit 1 - fi + # else + # echo "This script needs to be run as root or via sudo to install packages." + # exit 1 + # fi + else + echo "Python TK found..." fi elif [ "$distro" = "None" ] || [ "$family" = "None" ]; then if [ "$distro" = "None" ]; then diff --git a/tools/check_local_modules.py b/setup/check_local_modules.py similarity index 100% rename from tools/check_local_modules.py rename to setup/check_local_modules.py diff --git a/tools/create_user_files.py b/setup/create_user_files.py similarity index 100% rename from tools/create_user_files.py rename to setup/create_user_files.py diff --git a/tools/debug_info.py b/setup/debug_info.py similarity index 100% rename from tools/debug_info.py rename to setup/debug_info.py diff --git a/tools/setup_windows.py b/setup/setup_common.py similarity index 69% rename from tools/setup_windows.py rename to setup/setup_common.py index 6f00cd2ae..aa9e9c7a1 100644 --- a/tools/setup_windows.py +++ b/setup/setup_common.py @@ -1,5 +1,6 @@ import subprocess import os +import re import sys import filecmp import logging @@ -12,10 +13,6 @@ errors = 0 # Define the 'errors' variable before using it log = logging.getLogger('sd') -# ANSI escape code for yellow color -YELLOW = '\033[93m' -RESET_COLOR = '\033[0m' - # setup console and file logging def setup_logging(clean=False): # @@ -258,27 +255,6 @@ def git(arg: str, folder: str = None, ignore: bool = False): log.debug(f'Git output: {txt}') return txt -def cudann_install(): - cudnn_src = os.path.join( - os.path.dirname(os.path.realpath(__file__)), '..\cudnn_windows' - ) - cudnn_dest = os.path.join(sysconfig.get_paths()['purelib'], 'torch', 'lib') - - log.info(f'Checking for CUDNN files in {cudnn_dest}...') - if os.path.exists(cudnn_src): - if os.path.exists(cudnn_dest): - # check for different files - filecmp.clear_cache() - for file in os.listdir(cudnn_src): - src_file = os.path.join(cudnn_src, file) - dest_file = os.path.join(cudnn_dest, file) - # if dest file exists, check if it's different - if os.path.exists(dest_file): - shutil.copy2(src_file, cudnn_dest) - log.info('Copied CUDNN 8.6 files to destination') - else: - log.error(f'Installation Failed: "{cudnn_src}" could not be found. ') - def pip(arg: str, ignore: bool = False, quiet: bool = False): arg = arg.replace('>=', '==') @@ -302,6 +278,10 @@ def installed(package, friendly: str = None): # # This function was adapted from code written by vladimandic: https://github.com/vladmandic/automatic/commits/master # + + # Remove brackets and their contents from the line using regular expressions + # eg diffusers[torch]==0.10.2 becomes diffusers==0.10.2 + package = re.sub(r'\[.*?\]', '', package) ok = True try: @@ -369,6 +349,27 @@ def install( pip(f'install --upgrade {package}', ignore=ignore) +def install_requirements(requirements_file): + log.info(f'Verifying requirements against {requirements_file}...') + with open(requirements_file, 'r', encoding='utf8') as f: + # Read lines from the requirements file, strip whitespace, and filter out empty lines, comments, and lines starting with '.' + lines = [ + line.strip() + for line in f.readlines() + if line.strip() != '' + and not line.startswith('#') + and line is not None + and not line.startswith('.') + ] + + # Iterate over each line and install the requirements + for line in lines: + # Remove brackets and their contents from the line using regular expressions + # eg diffusers[torch]==0.10.2 becomes diffusers==0.10.2 + package_name = re.sub(r'\[.*?\]', '', line) + install(line, package_name) + + def ensure_base_requirements(): try: import rich # pylint: disable=unused-import @@ -417,24 +418,6 @@ def delete_file(file_path): os.remove(file_path) -def install_requirements(requirements_file): - # - # This function was adapted from code written by vladimandic: https://github.com/vladmandic/automatic/commits/master - # - - log.info('Verifying requirements') - with open(requirements_file, 'r', encoding='utf8') as f: - lines = [ - line.strip() - for line in f.readlines() - if line.strip() != '' - and not line.startswith('#') - and line is not None - ] - for line in lines: - install(line) - - def write_to_file(file_path, content): try: with open(file_path, 'w') as file: @@ -444,114 +427,6 @@ def write_to_file(file_path, content): print(f'Error: {e}') -def sync_bits_and_bytes_files(): - import filecmp - - """ - Check for "different" bitsandbytes Files and copy only if necessary. - This function is specific for Windows OS. - """ - - # Only execute on Windows - if os.name != 'nt': - print('This function is only applicable to Windows OS.') - return - - try: - log.info(f'Copying bitsandbytes files...') - # Define source and destination directories - source_dir = os.path.join(os.getcwd(), 'bitsandbytes_windows') - - dest_dir_base = os.path.join( - sysconfig.get_paths()['purelib'], 'bitsandbytes' - ) - - # Clear file comparison cache - filecmp.clear_cache() - - # Iterate over each file in source directory - for file in os.listdir(source_dir): - source_file_path = os.path.join(source_dir, file) - - # Decide the destination directory based on file name - if file in ('main.py', 'paths.py'): - dest_dir = os.path.join(dest_dir_base, 'cuda_setup') - else: - dest_dir = dest_dir_base - - dest_file_path = os.path.join(dest_dir, file) - - # Compare the source file with the destination file - if os.path.exists(dest_file_path) and filecmp.cmp( - source_file_path, dest_file_path - ): - log.debug( - f'Skipping {source_file_path} as it already exists in {dest_dir}' - ) - else: - # Copy file from source to destination, maintaining original file's metadata - log.debug(f'Copy {source_file_path} to {dest_dir}') - shutil.copy2(source_file_path, dest_dir) - - except FileNotFoundError as fnf_error: - log.error(f'File not found error: {fnf_error}') - except PermissionError as perm_error: - log.error(f'Permission error: {perm_error}') - except Exception as e: - log.error(f'An unexpected error occurred: {e}') - - -def install_kohya_ss_torch1(): - check_repo_version() - check_python() - - # Upgrade pip if needed - install('--upgrade pip') - - if check_torch() == 2: - input( - f'{YELLOW}\nTorch 2 is already installed in the venv. To install Torch 1 delete the venv and re-run setup.bat\n\nHit any key to acknowledge.{RESET_COLOR}' - ) - return - - install( - 'torch==1.12.1+cu116 torchvision==0.13.1+cu116 --index-url https://download.pytorch.org/whl/cu116', - 'torch torchvision' - ) - install( - 'https://github.com/C43H66N12O12S2/stable-diffusion-webui/releases/download/f/xformers-0.0.14.dev0-cp310-cp310-win_amd64.whl -U -I --no-deps', - 'xformers-0.0.14' - ) - install_requirements('requirements_windows_torch1.txt') - sync_bits_and_bytes_files() - configure_accelerate() - # run_cmd(f'accelerate config') - - -def install_kohya_ss_torch2(): - check_repo_version() - check_python() - - # Upgrade pip if needed - install('--upgrade pip') - - if check_torch() == 1: - input( - f'{YELLOW}\nTorch 1 is already installed in the venv. To install Torch 2 delete the venv and re-run setup.bat\n\nHit any key to acknowledge.{RESET_COLOR}' - ) - return - - install( - 'torch==2.0.1+cu118 torchvision==0.15.2+cu118 --index-url https://download.pytorch.org/whl/cu118', - 'torch torchvision' - ) - install_requirements('requirements_windows_torch2.txt') - # install('https://huggingface.co/r4ziel/xformers_pre_built/resolve/main/triton-2.0.0-cp310-cp310-win_amd64.whl', 'triton', reinstall=reinstall) - sync_bits_and_bytes_files() - configure_accelerate() - # run_cmd(f'accelerate config') - - def clear_screen(): # Check the current operating system to execute the correct clear screen command if os.name == 'nt': # If the operating system is Windows @@ -559,52 +434,3 @@ def clear_screen(): else: # If the operating system is Linux or Mac os.system('clear') - -def main_menu(): - clear_screen() - while True: - print('\nKohya_ss GUI setup menu:\n') - print('1. Install kohya_ss gui') - print('2. Install cudann files') - print('3. Manually configure accelerate') - print('4. Start Kohya_ss GUI in browser') - print('5. Quit') - - choice = input('\nEnter your choice: ') - print('') - - if choice == '1': - while True: - print('1. Torch 1') - print('2. Torch 2') - print('3. Cancel') - choice_torch = input('\nEnter your choice: ') - print('') - - if choice_torch == '1': - install_kohya_ss_torch1() - break - elif choice_torch == '2': - install_kohya_ss_torch2() - break - elif choice_torch == '3': - break - else: - print('Invalid choice. Please enter a number between 1-3.') - elif choice == '2': - cudann_install() - elif choice == '3': - run_cmd('accelerate config') - elif choice == '4': - subprocess.Popen('start cmd /c .\gui.bat --inbrowser', shell=True) - elif choice == '5': - print('Quitting the program.') - break - else: - print('Invalid choice. Please enter a number between 1-5.') - - -if __name__ == '__main__': - ensure_base_requirements() - setup_logging() - main_menu() diff --git a/setup/setup_windows.py b/setup/setup_windows.py new file mode 100644 index 000000000..d1a92b5ed --- /dev/null +++ b/setup/setup_windows.py @@ -0,0 +1,194 @@ +import subprocess +import os +import filecmp +import logging +import shutil +import sysconfig +import setup_common + +errors = 0 # Define the 'errors' variable before using it +log = logging.getLogger('sd') + +# ANSI escape code for yellow color +YELLOW = '\033[93m' +RESET_COLOR = '\033[0m' + + +def cudann_install(): + cudnn_src = os.path.join( + os.path.dirname(os.path.realpath(__file__)), '..\cudnn_windows' + ) + cudnn_dest = os.path.join(sysconfig.get_paths()['purelib'], 'torch', 'lib') + + log.info(f'Checking for CUDNN files in {cudnn_dest}...') + if os.path.exists(cudnn_src): + if os.path.exists(cudnn_dest): + # check for different files + filecmp.clear_cache() + for file in os.listdir(cudnn_src): + src_file = os.path.join(cudnn_src, file) + dest_file = os.path.join(cudnn_dest, file) + # if dest file exists, check if it's different + if os.path.exists(dest_file): + shutil.copy2(src_file, cudnn_dest) + log.info('Copied CUDNN 8.6 files to destination') + else: + log.error(f'Installation Failed: "{cudnn_src}" could not be found. ') + + +def sync_bits_and_bytes_files(): + import filecmp + + """ + Check for "different" bitsandbytes Files and copy only if necessary. + This function is specific for Windows OS. + """ + + # Only execute on Windows + if os.name != 'nt': + print('This function is only applicable to Windows OS.') + return + + try: + log.info(f'Copying bitsandbytes files...') + # Define source and destination directories + source_dir = os.path.join(os.getcwd(), 'bitsandbytes_windows') + + dest_dir_base = os.path.join( + sysconfig.get_paths()['purelib'], 'bitsandbytes' + ) + + # Clear file comparison cache + filecmp.clear_cache() + + # Iterate over each file in source directory + for file in os.listdir(source_dir): + source_file_path = os.path.join(source_dir, file) + + # Decide the destination directory based on file name + if file in ('main.py', 'paths.py'): + dest_dir = os.path.join(dest_dir_base, 'cuda_setup') + else: + dest_dir = dest_dir_base + + dest_file_path = os.path.join(dest_dir, file) + + # Compare the source file with the destination file + if os.path.exists(dest_file_path) and filecmp.cmp( + source_file_path, dest_file_path + ): + log.debug( + f'Skipping {source_file_path} as it already exists in {dest_dir}' + ) + else: + # Copy file from source to destination, maintaining original file's metadata + log.debug(f'Copy {source_file_path} to {dest_dir}') + shutil.copy2(source_file_path, dest_dir) + + except FileNotFoundError as fnf_error: + log.error(f'File not found error: {fnf_error}') + except PermissionError as perm_error: + log.error(f'Permission error: {perm_error}') + except Exception as e: + log.error(f'An unexpected error occurred: {e}') + + +def install_kohya_ss_torch1(): + setup_common.check_repo_version() + setup_common.check_python() + + # Upgrade pip if needed + setup_common.install('--upgrade pip') + + if setup_common.check_torch() == 2: + input( + f'{YELLOW}\nTorch 2 is already installed in the venv. To install Torch 1 delete the venv and re-run setup.bat\n\nHit enter to continue...{RESET_COLOR}' + ) + return + + setup_common.install( + 'torch==1.12.1+cu116 torchvision==0.13.1+cu116 --index-url https://download.pytorch.org/whl/cu116', + 'torch torchvision' + ) + setup_common.install( + 'https://github.com/C43H66N12O12S2/stable-diffusion-webui/releases/download/f/xformers-0.0.14.dev0-cp310-cp310-win_amd64.whl -U -I --no-deps', + 'xformers-0.0.14' + ) + setup_common.install_requirements('requirements_windows_torch1.txt') + sync_bits_and_bytes_files() + setup_common.configure_accelerate() + # run_cmd(f'accelerate config') + + +def install_kohya_ss_torch2(): + setup_common.check_repo_version() + setup_common.check_python() + + # Upgrade pip if needed + setup_common.install('--upgrade pip') + + if setup_common.check_torch() == 1: + input( + f'{YELLOW}\nTorch 1 is already installed in the venv. To install Torch 2 delete the venv and re-run setup.bat\n\nHit any key to acknowledge.{RESET_COLOR}' + ) + return + + setup_common.install( + 'torch==2.0.1+cu118 torchvision==0.15.2+cu118 --index-url https://download.pytorch.org/whl/cu118', + 'torch torchvision' + ) + setup_common.install_requirements('requirements_windows_torch2.txt') + # install('https://huggingface.co/r4ziel/xformers_pre_built/resolve/main/triton-2.0.0-cp310-cp310-win_amd64.whl', 'triton', reinstall=reinstall) + sync_bits_and_bytes_files() + setup_common.configure_accelerate() + # run_cmd(f'accelerate config') + + +def main_menu(): + setup_common.clear_screen() + while True: + print('\nKohya_ss GUI setup menu:\n') + print('1. Install kohya_ss gui') + print('2. Install cudann files') + print('3. Manually configure accelerate') + print('4. Start Kohya_ss GUI in browser') + print('5. Quit') + + choice = input('\nEnter your choice: ') + print('') + + if choice == '1': + while True: + print('1. Torch 1') + print('2. Torch 2') + print('3. Cancel') + choice_torch = input('\nEnter your choice: ') + print('') + + if choice_torch == '1': + install_kohya_ss_torch1() + break + elif choice_torch == '2': + install_kohya_ss_torch2() + break + elif choice_torch == '3': + break + else: + print('Invalid choice. Please enter a number between 1-3.') + elif choice == '2': + cudann_install() + elif choice == '3': + setup_common.run_cmd('accelerate config') + elif choice == '4': + subprocess.Popen('start cmd /k .\gui.bat --inbrowser', shell=True) # /k keep the terminal open on quit. /c would close the terminal instead + elif choice == '5': + print('Quitting the program.') + break + else: + print('Invalid choice. Please enter a number between 1-5.') + + +if __name__ == '__main__': + setup_common.ensure_base_requirements() + setup_common.setup_logging() + main_menu() diff --git a/tools/update_bitsandbytes.py b/setup/update_bitsandbytes.py similarity index 100% rename from tools/update_bitsandbytes.py rename to setup/update_bitsandbytes.py diff --git a/tools/validate_requirements_unix.py b/setup/validate_requirements.py similarity index 84% rename from tools/validate_requirements_unix.py rename to setup/validate_requirements.py index c09f91b5a..93f128bf9 100644 --- a/tools/validate_requirements_unix.py +++ b/setup/validate_requirements.py @@ -3,15 +3,14 @@ import sys import shutil import argparse -import subprocess -from setup_windows import check_repo_version +import setup_common # Get the absolute path of the current file's directory (Kohua_SS project directory) project_directory = os.path.dirname(os.path.abspath(__file__)) -# Check if the "tools" directory is present in the project_directory -if "tools" in project_directory: - # If the "tools" directory is present, move one level up to the parent directory +# Check if the "setup" directory is present in the project_directory +if "setup" in project_directory: + # If the "setup" directory is present, move one level up to the parent directory project_directory = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # Add the project directory to the beginning of the Python search path @@ -22,7 +21,6 @@ # Set up logging log = setup_logging() - def check_torch(): # Check for nVidia toolkit or AMD toolkit if shutil.which('nvidia-smi') is not None or os.path.exists( @@ -73,13 +71,8 @@ def check_torch(): sys.exit(1) -def install_requirements(requirements_file): - log.info('Verifying requirements') - subprocess.run(f'"{sys.executable}" -m pip install -U -r "{requirements_file}"', shell=True, check=False, env=os.environ) - - def main(): - check_repo_version() + setup_common.check_repo_version() # Parse command line arguments parser = argparse.ArgumentParser( description='Validate that requirements are satisfied.' @@ -93,7 +86,13 @@ def main(): parser.add_argument('--debug', action='store_true', help='Debug on') args = parser.parse_args() - install_requirements(args.requirements) + if not args.requirements: + if check_torch() == 1: + setup_common.install_requirements('requirements_windows_torch1.txt') + else: + setup_common.install_requirements('requirements_windows_torch2.txt') + else: + setup_common.install_requirements(args.requirements) if __name__ == '__main__': diff --git a/tools/cudann_1.8_install.py b/tools/cudann_1.8_install.py deleted file mode 100644 index 04f96fd52..000000000 --- a/tools/cudann_1.8_install.py +++ /dev/null @@ -1,26 +0,0 @@ -import filecmp -import os -import shutil -import sys -import sysconfig - -# Check for "different" B&B Files and copy only if necessary -if os.name == "nt": - python = sys.executable - cudnn_src = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..\cudnn_windows") - cudnn_dest = os.path.join(sysconfig.get_paths()["purelib"], "torch", "lib") - - print(f"Checking for CUDNN files in {cudnn_dest}") - if os.path.exists(cudnn_src): - if os.path.exists(cudnn_dest): - # check for different files - filecmp.clear_cache() - for file in os.listdir(cudnn_src): - src_file = os.path.join(cudnn_src, file) - dest_file = os.path.join(cudnn_dest, file) - #if dest file exists, check if it's different - if os.path.exists(dest_file): - shutil.copy2(src_file, cudnn_dest) - print("Copied CUDNN 8.6 files to destination") - else: - print(f"Installation Failed: \"{cudnn_src}\" could not be found. ") diff --git a/tools/validate_requirements.py b/tools/validate_requirements.py deleted file mode 100644 index 4117119d5..000000000 --- a/tools/validate_requirements.py +++ /dev/null @@ -1,122 +0,0 @@ -import os -import re -import sys -import shutil -import argparse -from setup_windows import install, check_repo_version - -# Get the absolute path of the current file's directory (Kohua_SS project directory) -project_directory = os.path.dirname(os.path.abspath(__file__)) - -# Check if the "tools" directory is present in the project_directory -if "tools" in project_directory: - # If the "tools" directory is present, move one level up to the parent directory - project_directory = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - -# Add the project directory to the beginning of the Python search path -sys.path.insert(0, project_directory) - -from library.custom_logging import setup_logging - -# Set up logging -log = setup_logging() - - -def check_torch(): - # Check for nVidia toolkit or AMD toolkit - if shutil.which('nvidia-smi') is not None or os.path.exists( - os.path.join( - os.environ.get('SystemRoot') or r'C:\Windows', - 'System32', - 'nvidia-smi.exe', - ) - ): - log.info('nVidia toolkit detected') - elif shutil.which('rocminfo') is not None or os.path.exists( - '/opt/rocm/bin/rocminfo' - ): - log.info('AMD toolkit detected') - else: - log.info('Using CPU-only Torch') - - try: - import torch - - log.info(f'Torch {torch.__version__}') - - # Check if CUDA is available - if not torch.cuda.is_available(): - log.warning('Torch reports CUDA not available') - else: - if torch.version.cuda: - # Log nVidia CUDA and cuDNN versions - log.info( - f'Torch backend: nVidia CUDA {torch.version.cuda} cuDNN {torch.backends.cudnn.version() if torch.backends.cudnn.is_available() else "N/A"}' - ) - elif torch.version.hip: - # Log AMD ROCm HIP version - log.info(f'Torch backend: AMD ROCm HIP {torch.version.hip}') - else: - log.warning('Unknown Torch backend') - - # Log information about detected GPUs - for device in [ - torch.cuda.device(i) for i in range(torch.cuda.device_count()) - ]: - log.info( - f'Torch detected GPU: {torch.cuda.get_device_name(device)} VRAM {round(torch.cuda.get_device_properties(device).total_memory / 1024 / 1024)} Arch {torch.cuda.get_device_capability(device)} Cores {torch.cuda.get_device_properties(device).multi_processor_count}' - ) - return int(torch.__version__[0]) - except Exception as e: - log.error(f'Could not load torch: {e}') - sys.exit(1) - - -def install_requirements(requirements_file): - log.info('Verifying requirements') - with open(requirements_file, 'r', encoding='utf8') as f: - # Read lines from the requirements file, strip whitespace, and filter out empty lines, comments, and lines starting with '.' - lines = [ - line.strip() - for line in f.readlines() - if line.strip() != '' - and not line.startswith('#') - and line is not None - and not line.startswith('.') - ] - - # Iterate over each line and install the requirements - for line in lines: - # Remove brackets and their contents from the line using regular expressions - # eg diffusers[torch]==0.10.2 becomes diffusers==0.10.2 - package_name = re.sub(r'\[.*?\]', '', line) - install(line, package_name) - - -def main(): - check_repo_version() - # Parse command line arguments - parser = argparse.ArgumentParser( - description='Validate that requirements are satisfied.' - ) - parser.add_argument( - '-r', - '--requirements', - type=str, - help='Path to the requirements file.', - ) - parser.add_argument('--debug', action='store_true', help='Debug on') - args = parser.parse_args() - - if not args.requirements: - # Check Torch - if check_torch() == 1: - install_requirements('requirements_windows_torch1.txt') - else: - install_requirements('requirements_windows_torch2.txt') - else: - install_requirements(args.requirements) - - -if __name__ == '__main__': - main() diff --git a/upgrade.bat b/upgrade.bat index 0fdf50360..3dd329145 100644 --- a/upgrade.bat +++ b/upgrade.bat @@ -13,4 +13,4 @@ git pull call .\venv\Scripts\activate.bat :: Validate requirements -python.exe .\tools\validate_requirements.py +python.exe .\setup\validate_requirements.py diff --git a/upgrade.sh b/upgrade.sh deleted file mode 100644 index 6c2f39f41..000000000 --- a/upgrade.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -# Check if there are any changes that need to be committed -if git status --short | grep -q "^[^ ?][^?]*"; then - echo "There are changes that need to be committed. Please stash or undo your changes before running this script." - exit 1 -fi - -# Pull the latest changes from the remote repository -git pull - -# Activate the virtual environment -source venv/bin/activate - -# Upgrade the required packages -pip install --use-pep517 --upgrade -r requirements_unix.txt