diff --git a/.gitignore b/.gitignore index d2be885e4..ccb515218 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,7 @@ setup.log logs SmilingWolf uninstall.txt +test/output +test/logs +test/*.json +test/ft diff --git a/README.md b/README.md index e16099fc5..4107ce580 100644 --- a/README.md +++ b/README.md @@ -345,6 +345,10 @@ This will store a backup file with your current locally installed pip packages a ## Change History +* 2023/06/03 (v21.6.4) +- Fix dreambooth issue with new logging +- Update setup and upgrade scripts +- Adding test folder * 2023/06/02 (v21.6.3) - Fix install issue with setup logic * 2023/06/02 (v21.6.2) diff --git a/activate.bat b/activate.bat deleted file mode 100644 index 20c21a715..000000000 --- a/activate.bat +++ /dev/null @@ -1,5 +0,0 @@ -@echo off - -call venv\Scripts\activate.bat - -pause \ No newline at end of file diff --git a/dreambooth_gui.py b/dreambooth_gui.py index 09bc73a11..d69c3b1f2 100644 --- a/dreambooth_gui.py +++ b/dreambooth_gui.py @@ -412,11 +412,7 @@ def train_model( # Check if subfolders are present. If not let the user know and return if not subfolders: - log.info( - '\033[33mNo subfolders were found in', - train_data_dir, - " can't train\...033[0m", - ) + log.info(f'No {subfolders} were found in train_data_dir can\'t train...') return total_steps = 0 @@ -427,11 +423,7 @@ def train_model( try: repeats = int(folder.split('_')[0]) except ValueError: - log.info( - '\033[33mSubfolder', - folder, - "does not have a proper repeat value, please correct the name or remove it... can't train...\033[0m", - ) + log.info(f'Subfolder {folder} does not have a proper repeat value, please correct the name or remove it... can\'t train...') continue # Count the number of images in the folder @@ -456,14 +448,10 @@ def train_model( total_steps += steps # Print the result - log.info('\033[33mFolder', folder, ':', steps, 'steps\033[0m') + log.info(f'Folder {folder} : steps {steps}') if total_steps == 0: - log.info( - '\033[33mNo images were found in folder', - train_data_dir, - '... please rectify!\033[0m', - ) + log.info(f'No images were found in folder {train_data_dir}... please rectify!') return # Print the result @@ -472,8 +460,7 @@ def train_model( if reg_data_dir == '': reg_factor = 1 else: - log.info( - '\033[94mRegularisation images are used... Will double the number of steps required...\033[0m' + log.info(f'Regularisation images are used... Will double the number of steps required...' ) reg_factor = 2 diff --git a/gui.bat b/gui.bat index 34b7bcb16..2cdff96e1 100644 --- a/gui.bat +++ b/gui.bat @@ -1,36 +1,9 @@ @echo off -setlocal - -REM Define color variables -set "yellow_text=" -set "blue_text=" -set "reset_text=" - :: Deactivate the virtual environment call .\venv\Scripts\deactivate.bat -REM Run pip freeze and capture the output -for /f "delims=" %%I in ('pip freeze') do ( - set "pip_output=%%I" - goto :CheckModules -) - -:CheckModules -REM Check if modules are found in the output -if defined pip_output ( - echo %yellow_text%============================================================= - echo Modules installed outside the virtual environment were found. - echo This can cause issues. Please review the installed modules. - echo. - echo You can uninstall all local modules with: - echo. - echo %blue_text%deactivate - echo pip freeze ^> uninstall.txt - echo pip uninstall -y -r uninstall.txt - echo %yellow_text%=============================================================%reset_text% -) - -endlocal +:: Calling external python program to check for local modules +python .\tools\check_local_modules.py --no_question :: Activate the virtual environment call .\venv\Scripts\activate.bat @@ -39,8 +12,15 @@ set PATH=%PATH%;%~dp0venv\Lib\site-packages\torch\lib :: Debug info about system :: python.exe .\tools\debug_info.py -:: Validate the requirements and store the exit code -python.exe .\tools\validate_requirements.py +:: Check if torch_version is 1 +findstr /C:"1" ".\logs\status\torch_version" >nul + +:: Check the error level to determine if the text was found +if %errorlevel% equ 0 ( + python.exe .\tools\validate_requirements.py -r requirements_windows_torch1.txt +) else ( + python.exe .\tools\validate_requirements.py -r requirements_windows_torch2.txt +) :: If the exit code is 0, run the kohya_gui.py script with the command-line arguments if %errorlevel% equ 0 ( diff --git a/requirements.txt b/requirements.txt index 6f1fa8774..62bdde789 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,39 +1,31 @@ -# Some comments -accelerate==0.18.0 +accelerate==0.15.0 albumentations==1.3.0 altair==4.2.2 -# https://github.com/bmaltais/bitsandbytes-windows-webui/raw/main/bitsandbytes-0.38.1-py3-none-any.whl; sys_platform == 'win32' -# This next line is not an error but rather there to properly catch if the url based bitsandbytes was properly installed by the line above... -bitsandbytes==0.35.0; sys_platform == 'win32' -bitsandbytes==0.35.0; (sys_platform == "darwin" or sys_platform == "linux") +bitsandbytes==0.35.0 dadaptation==3.1 diffusers[torch]==0.10.2 easygui==0.98.3 einops==0.6.0 +fairscale==0.4.13 ftfy==6.1.1 -gradio==3.32.0; sys_platform != 'darwin' gradio==3.23.0; sys_platform == 'darwin' +gradio==3.32.0; sys_platform != 'darwin' +huggingface-hub==0.13.0; sys_platform == 'darwin' +huggingface-hub==0.13.3; sys_platform != 'darwin' lion-pytorch==0.0.6 +lycoris_lora==0.1.4 opencv-python==4.7.0.68 pytorch-lightning==1.9.0 rich==13.4.1 safetensors==0.2.6 tensorboard==2.10.1 ; sys_platform != 'darwin' tensorboard==2.12.1 ; sys_platform == 'darwin' +tensorflow==2.10.1; sys_platform != 'darwin' +timm==0.6.12 tk==0.1.0 toml==0.10.2 transformers==4.26.0 voluptuous==0.13.1 wandb==0.15.0 -# for BLIP captioning -fairscale==0.4.13 -requests==2.28.2 -timm==0.6.12 -# tensorflow<2.11 -huggingface-hub==0.13.3; sys_platform != 'darwin' -huggingface-hub==0.13.0; sys_platform == 'darwin' -tensorflow==2.10.1; sys_platform != 'darwin' -# For locon support -lycoris_lora==0.1.4 # for kohya_ss library . diff --git a/requirements_windows_torch1.txt b/requirements_windows_torch1.txt new file mode 100644 index 000000000..599f6f4c2 --- /dev/null +++ b/requirements_windows_torch1.txt @@ -0,0 +1,28 @@ +accelerate==0.15.0 +albumentations==1.3.0 +altair==4.2.2 +bitsandbytes==0.35.0 +dadaptation==3.1 +diffusers[torch]==0.10.2 +easygui==0.98.3 +einops==0.6.0 +fairscale==0.4.13 +ftfy==6.1.1 +gradio==3.32.0 +huggingface-hub==0.13.3 +lion-pytorch==0.0.6 +lycoris_lora==0.1.4 +opencv-python==4.7.0.68 +pytorch-lightning==1.9.0 +rich==13.4.1 +safetensors==0.2.6 +tensorboard==2.10.1 +tensorflow==2.10.1 +timm==0.6.12 +tk==0.1.0 +toml==0.10.2 +transformers==4.26.0 +voluptuous==0.13.1 +wandb==0.15.0 +# for kohya_ss library +. diff --git a/requirements_windows_torch2.txt b/requirements_windows_torch2.txt new file mode 100644 index 000000000..cfc6df8e1 --- /dev/null +++ b/requirements_windows_torch2.txt @@ -0,0 +1,28 @@ +accelerate==0.18.0 +albumentations==1.3.0 +altair==4.2.2 +bitsandbytes==0.35.0 +dadaptation==3.1 +diffusers[torch]==0.10.2 +easygui==0.98.3 +einops==0.6.0 +fairscale==0.4.13 +ftfy==6.1.1 +gradio==3.33.1 +huggingface-hub==0.15.1 +lion-pytorch==0.0.6 +lycoris_lora==0.1.4 +opencv-python==4.7.0.68 +pytorch-lightning==1.9.0 +rich==13.4.1 +safetensors==0.2.6 +tensorboard==2.12.3 +tensorflow==2.12.0 +timm==0.6.12 +tk==0.1.0 +toml==0.10.2 +transformers==4.26.0 +voluptuous==0.13.1 +wandb==0.15.0 +# for kohya_ss library +. diff --git a/setup.bat b/setup.bat index 2635330c9..6fa3696e7 100644 --- a/setup.bat +++ b/setup.bat @@ -1,119 +1,56 @@ @echo off -setlocal - -REM Define color variables -set "yellow_text=" -set "blue_text=" -set "reset_text=" set PYTHON_VER=3.10.9 -REM Check if Python version meets the recommended version +:: Check if Python version meets the recommended version python --version 2>nul | findstr /b /c:"Python %PYTHON_VER%" >nul if errorlevel 1 ( echo Warning: Python version %PYTHON_VER% is recommended. ) IF NOT EXIST venv ( + echo Creating venv... python -m venv venv -) ELSE ( - echo venv folder already exists, skipping creation... ) +:: Create the directory if it doesn't exist +mkdir ".\logs\status" > nul 2>&1 + :: Deactivate the virtual environment call .\venv\Scripts\deactivate.bat -REM Run pip freeze and capture the output -for /f "delims=" %%I in ('pip freeze') do ( - set "pip_output=%%I" - goto :CheckModules +:: Calling external python program to check for local modules +python .\tools\check_local_modules.py +if %errorlevel% equ 1 ( + exit /b ) -:CheckModules -REM Check if modules are found in the output -if defined pip_output ( - echo %yellow_text%============================================================= - echo Modules installed outside the virtual environment were found. - echo This can cause issues. Please review the installed modules. - echo. - echo You can uninstall all local modules with: - echo. - echo %blue_text%deactivate - echo pip freeze ^> uninstall.txt - echo pip uninstall -y -r uninstall.txt - echo %yellow_text%=============================================================%reset_text% - goto :PromptUser -) - -:PromptUser -REM Ask the user if they want to quit or continue -echo. -echo Do you want to: -echo [1] - Quit -echo [2] - Continue with the setup -set /p "choice=Enter your choice (1 or 2): " -if /i "%choice%"=="1" ( - echo You chose to quit. - echo. - goto :Quit -) else if /i "%choice%"=="2" ( - echo You chose to continue. - echo. - goto :ContinueSetup -) else ( - echo Invalid choice. Please try again. - echo. - goto :PromptUser -) - -:ContinueSetup -endlocal - call .\venv\Scripts\activate.bat -REM Upgrade pip if needed -python.exe -m pip install --upgrade pip - -echo Do you want to uninstall previous versions of torch and associated files before installing? Usefull if you are upgrading from torch 1.12.1 to torch 2.0.0 or if you are downgrading from torch 2.0.0 to torch 1.12.1. -echo [1] - Yes -echo [2] - No (recommanded for most) -set /p uninstall_choice="Enter your choice (1 or 2): " - -if %uninstall_choice%==1 ( - pip uninstall -y xformers torch torchvision triton -) +:: Upgrade pip if needed +pip install --upgrade pip +echo. echo Please choose the version of torch you want to install: -echo [1] - v1 (torch 1.12.1) (Recommended) -echo [2] - v2 (torch 2.0.0) (Experimental) +echo [1] - v1 (torch 1.12.1) (Recommended for best compatibility) +echo [2] - v2 (torch 2.0.0) (Experimental, faster but more prone to issues) set /p choice="Enter your choice (1 or 2): " -:: Only does this section to cleanup the old custom dll versions that we used to use. No longer needed now with the new bitsandbytes version -pip uninstall -y bitsandbytes -IF EXIST ".\venv\Lib\site-packages\bitsandbytes" ( - rmdir .\venv\Lib\site-packages\bitsandbytes -) -::::::::::::::::::::::::: - if %choice%==1 ( pip install torch==1.12.1+cu116 torchvision==0.13.1+cu116 --extra-index-url https://download.pytorch.org/whl/cu116 - pip install --use-pep517 --upgrade -r requirements.txt + del ".\logs\status\torch_version" > nul 2>&1 + echo 1 > ".\logs\status\torch_version" + pip install --use-pep517 --upgrade -r requirements_windows_torch1.txt pip install -U -I --no-deps https://github.com/C43H66N12O12S2/stable-diffusion-webui/releases/download/f/xformers-0.0.14.dev0-cp310-cp310-win_amd64.whl ) else ( pip install torch==2.0.1+cu118 torchvision==0.15.2+cu118 --extra-index-url https://download.pytorch.org/whl/cu118 - pip install --use-pep517 --upgrade -r requirements.txt + del ".\logs\status\torch_version" > nul 2>&1 + echo 2 > ".\logs\status\torch_version" + pip install --use-pep517 --upgrade -r requirements_windows_torch2.txt pip install --upgrade xformers==0.0.20 - rem pip install -U -I --no-deps https://files.pythonhosted.org/packages/d6/f7/02662286419a2652c899e2b3d1913c47723fc164b4ac06a85f769c291013/xformers-0.0.17rc482-cp310-cp310-win_amd64.whl pip install https://huggingface.co/r4ziel/xformers_pre_built/resolve/main/triton-2.0.0-cp310-cp310-win_amd64.whl ) python.exe .\tools\update_bitsandbytes.py -@REM copy /y .\bitsandbytes_windows\*.dll .\venv\Lib\site-packages\bitsandbytes\ -@REM copy /y .\bitsandbytes_windows\cextension.py .\venv\Lib\site-packages\bitsandbytes\cextension.py -@REM copy /y .\bitsandbytes_windows\main.py .\venv\Lib\site-packages\bitsandbytes\cuda_setup\main.py - accelerate config - -:Quit -exit diff --git a/test/img/10_darius kawasaki person/Dariusz_Zawadzki.jpg b/test/img/10_darius kawasaki person/Dariusz_Zawadzki.jpg new file mode 100644 index 000000000..ccfc19947 Binary files /dev/null and b/test/img/10_darius kawasaki person/Dariusz_Zawadzki.jpg differ diff --git a/test/img/10_darius kawasaki person/Dariusz_Zawadzki.txt b/test/img/10_darius kawasaki person/Dariusz_Zawadzki.txt new file mode 100644 index 000000000..b589f7c7e --- /dev/null +++ b/test/img/10_darius kawasaki person/Dariusz_Zawadzki.txt @@ -0,0 +1 @@ +a painting of a steam punk skull with a gas mask , by darius kawasaki \ No newline at end of file diff --git a/test/img/10_darius kawasaki person/Dariusz_Zawadzki_2.jpg b/test/img/10_darius kawasaki person/Dariusz_Zawadzki_2.jpg new file mode 100644 index 000000000..bb062caca Binary files /dev/null and b/test/img/10_darius kawasaki person/Dariusz_Zawadzki_2.jpg differ diff --git a/test/img/10_darius kawasaki person/Dariusz_Zawadzki_2.txt b/test/img/10_darius kawasaki person/Dariusz_Zawadzki_2.txt new file mode 100644 index 000000000..e7ded5fa8 --- /dev/null +++ b/test/img/10_darius kawasaki person/Dariusz_Zawadzki_2.txt @@ -0,0 +1 @@ +a painting of a man with a skull on his head , by darius kawasaki \ No newline at end of file diff --git a/test/img/10_darius kawasaki person/Dariusz_Zawadzki_3.jpg b/test/img/10_darius kawasaki person/Dariusz_Zawadzki_3.jpg new file mode 100644 index 000000000..e84720572 Binary files /dev/null and b/test/img/10_darius kawasaki person/Dariusz_Zawadzki_3.jpg differ diff --git a/test/img/10_darius kawasaki person/Dariusz_Zawadzki_3.txt b/test/img/10_darius kawasaki person/Dariusz_Zawadzki_3.txt new file mode 100644 index 000000000..4ef297cf8 --- /dev/null +++ b/test/img/10_darius kawasaki person/Dariusz_Zawadzki_3.txt @@ -0,0 +1 @@ +a painting of a woman with a helmet on her head , by darius kawasaki \ No newline at end of file diff --git a/test/img/10_darius kawasaki person/Dariusz_Zawadzki_4.jpg b/test/img/10_darius kawasaki person/Dariusz_Zawadzki_4.jpg new file mode 100644 index 000000000..8821fa282 Binary files /dev/null and b/test/img/10_darius kawasaki person/Dariusz_Zawadzki_4.jpg differ diff --git a/test/img/10_darius kawasaki person/Dariusz_Zawadzki_4.txt b/test/img/10_darius kawasaki person/Dariusz_Zawadzki_4.txt new file mode 100644 index 000000000..4abb67e8f --- /dev/null +++ b/test/img/10_darius kawasaki person/Dariusz_Zawadzki_4.txt @@ -0,0 +1 @@ +a painting of a horned man with a goat head , by darius kawasaki \ No newline at end of file diff --git a/test/img/10_darius kawasaki person/Dariusz_Zawadzki_5.jpg b/test/img/10_darius kawasaki person/Dariusz_Zawadzki_5.jpg new file mode 100644 index 000000000..590a097a6 Binary files /dev/null and b/test/img/10_darius kawasaki person/Dariusz_Zawadzki_5.jpg differ diff --git a/test/img/10_darius kawasaki person/Dariusz_Zawadzki_5.txt b/test/img/10_darius kawasaki person/Dariusz_Zawadzki_5.txt new file mode 100644 index 000000000..406814f85 --- /dev/null +++ b/test/img/10_darius kawasaki person/Dariusz_Zawadzki_5.txt @@ -0,0 +1 @@ +a painting of a man playing a piano , by darius kawasaki \ No newline at end of file diff --git a/test/img/10_darius kawasaki person/Dariusz_Zawadzki_6.jpg b/test/img/10_darius kawasaki person/Dariusz_Zawadzki_6.jpg new file mode 100644 index 000000000..4a39ce428 Binary files /dev/null and b/test/img/10_darius kawasaki person/Dariusz_Zawadzki_6.jpg differ diff --git a/test/img/10_darius kawasaki person/Dariusz_Zawadzki_6.txt b/test/img/10_darius kawasaki person/Dariusz_Zawadzki_6.txt new file mode 100644 index 000000000..55de4a66e --- /dev/null +++ b/test/img/10_darius kawasaki person/Dariusz_Zawadzki_6.txt @@ -0,0 +1 @@ +a painting of a robot sitting on a rock , by darius kawasaki \ No newline at end of file diff --git a/test/img/10_darius kawasaki person/Dariusz_Zawadzki_7.jpg b/test/img/10_darius kawasaki person/Dariusz_Zawadzki_7.jpg new file mode 100644 index 000000000..0c1f50f42 Binary files /dev/null and b/test/img/10_darius kawasaki person/Dariusz_Zawadzki_7.jpg differ diff --git a/test/img/10_darius kawasaki person/Dariusz_Zawadzki_7.txt b/test/img/10_darius kawasaki person/Dariusz_Zawadzki_7.txt new file mode 100644 index 000000000..e36dca828 --- /dev/null +++ b/test/img/10_darius kawasaki person/Dariusz_Zawadzki_7.txt @@ -0,0 +1 @@ +a painting of a soldier with a helmet on , by darius kawasaki \ No newline at end of file diff --git a/test/img/10_darius kawasaki person/Dariusz_Zawadzki_8.jpg b/test/img/10_darius kawasaki person/Dariusz_Zawadzki_8.jpg new file mode 100644 index 000000000..7d3380be2 Binary files /dev/null and b/test/img/10_darius kawasaki person/Dariusz_Zawadzki_8.jpg differ diff --git a/test/img/10_darius kawasaki person/Dariusz_Zawadzki_8.txt b/test/img/10_darius kawasaki person/Dariusz_Zawadzki_8.txt new file mode 100644 index 000000000..0087fedfc --- /dev/null +++ b/test/img/10_darius kawasaki person/Dariusz_Zawadzki_8.txt @@ -0,0 +1 @@ +a painting of a giant crab with a large body , by darius kawasaki \ No newline at end of file diff --git a/tools/check_local_modules.py b/tools/check_local_modules.py new file mode 100644 index 000000000..860878eaa --- /dev/null +++ b/tools/check_local_modules.py @@ -0,0 +1,49 @@ +import argparse +import subprocess +import sys + +# Define color variables +yellow_text = "\033[1;33m" +blue_text = "\033[1;34m" +reset_text = "\033[0m" + +# Parse command line arguments +parser = argparse.ArgumentParser() +parser.add_argument('--no_question', action='store_true') +args = parser.parse_args() + +# Run pip freeze and capture the output +output = subprocess.getoutput("pip freeze") + +# Check if modules are found in the output +if output: + print(f"{yellow_text}=============================================================") + print("Modules installed outside the virtual environment were found.") + print("This can cause issues. Please review the installed modules.\n") + print("You can uninstall all local modules with:\n") + print(f"{blue_text}deactivate") + print("pip freeze > uninstall.txt") + print("pip uninstall -y -r uninstall.txt") + print(f"{yellow_text}============================================================={reset_text}") + + if args.no_question: + sys.exit(1) # Exit with code 1 for "no" without asking the user + + # Ask the user if they want to continue + valid_input = False + while not valid_input: + print('Do you want to continue?') + print('') + print('[1] - Yes') + print('[2] - No') + user_input = input("Enter your choice (1 or 2): ") + if user_input.lower() == "2": + valid_input = True + sys.exit(1) # Exit with code 1 for "no" + elif user_input.lower() == "1": + valid_input = True + sys.exit(0) # Exit with code 0 for "yes" + else: + print("Invalid input. Please enter '1' or '2'.") +else: + sys.exit(0) diff --git a/tools/validate_requirements.py b/tools/validate_requirements.py index 5a1072b3f..5e13e6174 100644 --- a/tools/validate_requirements.py +++ b/tools/validate_requirements.py @@ -103,8 +103,8 @@ def print_error_messages(missing_requirements, wrong_version_requirements, requi for requirement, expected_version, actual_version in wrong_version_requirements: log.info(f" - {requirement} (expected version {expected_version}, found version {actual_version})") - upgrade_script = "upgrade.ps1" if os.name == "nt" else "upgrade.sh" - log.info(f"\nRun \033[33m{upgrade_script}\033[0m or \033[33mpip install -U -r {requirements_file}\033[0m to resolve the missing requirements listed above...") + upgrade_script = "upgrade.bat" if os.name == "nt" else "upgrade.sh" + log.info(f"\nRun {upgrade_script} to resolve the missing requirements listed above...") sys.exit(1) log.info("All requirements satisfied.") diff --git a/upgrade.bat b/upgrade.bat index 50dfad57d..109694764 100644 --- a/upgrade.bat +++ b/upgrade.bat @@ -12,5 +12,14 @@ git pull :: Activate the virtual environment call .\venv\Scripts\activate.bat -:: Upgrade the required packages -pip install --use-pep517 --upgrade -r requirements.txt \ No newline at end of file +REM Check if torch_version is 1 +findstr /C:"1" ".\logs\status\torch_version" >nul + +REM Check the error level to determine if the text was found +if %errorlevel% equ 0 ( + echo Torch version 1... + pip install --use-pep517 --upgrade -r requirements_windows_torch1.txt +) else ( + echo Torch version 2... + pip install --use-pep517 --upgrade -r requirements_windows_torch2.txt +) diff --git a/upgrade.ps1 b/upgrade.ps1 deleted file mode 100644 index 267f0ccd4..000000000 --- a/upgrade.ps1 +++ /dev/null @@ -1,14 +0,0 @@ -# Check if there are any changes that need to be committed -if (git status --short) { - Write-Error "There are changes that need to be committed. Please stash or undo your changes before running this script." - return -} - -# Pull the latest changes from the remote repository -git pull - -# Activate the virtual environment -.\venv\Scripts\activate - -# Upgrade the required packages -pip install --use-pep517 --upgrade -r requirements.txt \ No newline at end of file