Merge pull request kohya-ss#900 from bmaltais/dev2

v21.6.4

.gitignore

@@ -15,3 +15,7 @@ setup.log
 logs
 SmilingWolf
 uninstall.txt
+test/output
+test/logs
+test/*.json
+test/ft

README.md

@@ -345,6 +345,10 @@ This will store a backup file with your current locally installed pip packages a
 
 ## Change History
 
+* 2023/06/03 (v21.6.4)
+- Fix dreambooth issue with new logging
+- Update setup and upgrade scripts
+- Adding test folder
 * 2023/06/02 (v21.6.3)
 - Fix install issue with setup logic
 * 2023/06/02 (v21.6.2)

dreambooth_gui.py

@@ -412,11 +412,7 @@ def train_model(
 
     # Check if subfolders are present. If not let the user know and return
     if not subfolders:
-        log.info(
-            '\033[33mNo subfolders were found in',
-            train_data_dir,
-            " can't train\...033[0m",
-        )
+        log.info(f'No {subfolders} were found in train_data_dir can\'t train...')
         return
 
     total_steps = 0
@@ -427,11 +423,7 @@ def train_model(
         try:
             repeats = int(folder.split('_')[0])
         except ValueError:
-            log.info(
-                '\033[33mSubfolder',
-                folder,
-                "does not have a proper repeat value, please correct the name or remove it... can't train...\033[0m",
-            )
+            log.info(f'Subfolder {folder} does not have a proper repeat value, please correct the name or remove it... can\'t train...')
             continue
 
         # Count the number of images in the folder
@@ -456,14 +448,10 @@ def train_model(
             total_steps += steps
 
             # Print the result
-            log.info('\033[33mFolder', folder, ':', steps, 'steps\033[0m')
+            log.info(f'Folder {folder} : steps {steps}')
 
     if total_steps == 0:
-        log.info(
-            '\033[33mNo images were found in folder',
-            train_data_dir,
-            '... please rectify!\033[0m',
-        )
+        log.info(f'No images were found in folder {train_data_dir}... please rectify!')
         return
 
     # Print the result
@@ -472,8 +460,7 @@ def train_model(
     if reg_data_dir == '':
         reg_factor = 1
     else:
-        log.info(
-            '\033[94mRegularisation images are used... Will double the number of steps required...\033[0m'
+        log.info(f'Regularisation images are used... Will double the number of steps required...'
         )
         reg_factor = 2
 

gui.bat

@@ -1,36 +1,9 @@
 @echo off
-setlocal
-
-REM Define color variables
-set "yellow_text="
-set "blue_text="
-set "reset_text="
-
 :: Deactivate the virtual environment
 call .\venv\Scripts\deactivate.bat
 
-REM Run pip freeze and capture the output
-for /f "delims=" %%I in ('pip freeze') do (
-    set "pip_output=%%I"
-    goto :CheckModules
-)
-
-:CheckModules
-REM Check if modules are found in the output
-if defined pip_output (
-    echo %yellow_text%=============================================================
-    echo Modules installed outside the virtual environment were found.
-    echo This can cause issues. Please review the installed modules.
-    echo.
-    echo You can uninstall all local modules with:
-    echo.
-    echo %blue_text%deactivate
-    echo pip freeze ^> uninstall.txt
-    echo pip uninstall -y -r uninstall.txt
-    echo %yellow_text%=============================================================%reset_text%
-)
-
-endlocal
+:: Calling external python program to check for local modules
+python .\tools\check_local_modules.py --no_question
 
 :: Activate the virtual environment
 call .\venv\Scripts\activate.bat
@@ -39,8 +12,15 @@ set PATH=%PATH%;%~dp0venv\Lib\site-packages\torch\lib
 :: Debug info about system
 :: python.exe .\tools\debug_info.py
 
-:: Validate the requirements and store the exit code
-python.exe .\tools\validate_requirements.py
+:: Check if torch_version is 1
+findstr /C:"1" ".\logs\status\torch_version" >nul
+
+:: Check the error level to determine if the text was found
+if %errorlevel% equ 0 (
+    python.exe .\tools\validate_requirements.py -r requirements_windows_torch1.txt
+) else (
+    python.exe .\tools\validate_requirements.py -r requirements_windows_torch2.txt
+)
 
 :: If the exit code is 0, run the kohya_gui.py script with the command-line arguments
 if %errorlevel% equ 0 (

requirements.txt

@@ -1,39 +1,31 @@
-# Some comments
-accelerate==0.18.0
+accelerate==0.15.0
 albumentations==1.3.0
 altair==4.2.2
-# https://github.com/bmaltais/bitsandbytes-windows-webui/raw/main/bitsandbytes-0.38.1-py3-none-any.whl; sys_platform == 'win32'
-# This next line is not an error but rather there to properly catch if the url based bitsandbytes was properly installed by the line above...
-bitsandbytes==0.35.0; sys_platform == 'win32'
-bitsandbytes==0.35.0; (sys_platform == "darwin" or sys_platform == "linux")
+bitsandbytes==0.35.0
 dadaptation==3.1
 diffusers[torch]==0.10.2
 easygui==0.98.3
 einops==0.6.0
+fairscale==0.4.13
 ftfy==6.1.1
-gradio==3.32.0; sys_platform != 'darwin'
 gradio==3.23.0; sys_platform == 'darwin'
+gradio==3.32.0; sys_platform != 'darwin'
+huggingface-hub==0.13.0; sys_platform == 'darwin'
+huggingface-hub==0.13.3; sys_platform != 'darwin'
 lion-pytorch==0.0.6
+lycoris_lora==0.1.4
 opencv-python==4.7.0.68
 pytorch-lightning==1.9.0
 rich==13.4.1
 safetensors==0.2.6
 tensorboard==2.10.1 ; sys_platform != 'darwin'
 tensorboard==2.12.1 ; sys_platform == 'darwin'
+tensorflow==2.10.1; sys_platform != 'darwin'
+timm==0.6.12
 tk==0.1.0
 toml==0.10.2
 transformers==4.26.0
 voluptuous==0.13.1
 wandb==0.15.0
-# for BLIP captioning
-fairscale==0.4.13
-requests==2.28.2
-timm==0.6.12
-# tensorflow<2.11
-huggingface-hub==0.13.3; sys_platform != 'darwin'
-huggingface-hub==0.13.0; sys_platform == 'darwin'
-tensorflow==2.10.1; sys_platform != 'darwin'
-# For locon support
-lycoris_lora==0.1.4
 # for kohya_ss library
 .

requirements_windows_torch1.txt

@@ -0,0 +1,28 @@
+accelerate==0.15.0
+albumentations==1.3.0
+altair==4.2.2
+bitsandbytes==0.35.0
+dadaptation==3.1
+diffusers[torch]==0.10.2
+easygui==0.98.3
+einops==0.6.0
+fairscale==0.4.13
+ftfy==6.1.1
+gradio==3.32.0
+huggingface-hub==0.13.3
+lion-pytorch==0.0.6
+lycoris_lora==0.1.4
+opencv-python==4.7.0.68
+pytorch-lightning==1.9.0
+rich==13.4.1
+safetensors==0.2.6
+tensorboard==2.10.1
+tensorflow==2.10.1
+timm==0.6.12
+tk==0.1.0
+toml==0.10.2
+transformers==4.26.0
+voluptuous==0.13.1
+wandb==0.15.0
+# for kohya_ss library
+.

requirements_windows_torch2.txt

@@ -0,0 +1,28 @@
+accelerate==0.18.0
+albumentations==1.3.0
+altair==4.2.2
+bitsandbytes==0.35.0
+dadaptation==3.1
+diffusers[torch]==0.10.2
+easygui==0.98.3
+einops==0.6.0
+fairscale==0.4.13
+ftfy==6.1.1
+gradio==3.33.1
+huggingface-hub==0.15.1
+lion-pytorch==0.0.6
+lycoris_lora==0.1.4
+opencv-python==4.7.0.68
+pytorch-lightning==1.9.0
+rich==13.4.1
+safetensors==0.2.6
+tensorboard==2.12.3
+tensorflow==2.12.0
+timm==0.6.12
+tk==0.1.0
+toml==0.10.2
+transformers==4.26.0
+voluptuous==0.13.1
+wandb==0.15.0
+# for kohya_ss library
+.

setup.bat

@@ -1,119 +1,56 @@
 @echo off
-setlocal
-
-REM Define color variables
-set "yellow_text="
-set "blue_text="
-set "reset_text="
 
 set PYTHON_VER=3.10.9
 
-REM Check if Python version meets the recommended version
+:: Check if Python version meets the recommended version
 python --version 2>nul | findstr /b /c:"Python %PYTHON_VER%" >nul
 if errorlevel 1 (
     echo Warning: Python version %PYTHON_VER% is recommended.
 )
 
 IF NOT EXIST venv (
+    echo Creating venv...
     python -m venv venv
-) ELSE (
-    echo venv folder already exists, skipping creation...
 )
 
+:: Create the directory if it doesn't exist
+mkdir ".\logs\status" > nul 2>&1
+
 :: Deactivate the virtual environment
 call .\venv\Scripts\deactivate.bat
 
-REM Run pip freeze and capture the output
-for /f "delims=" %%I in ('pip freeze') do (
-    set "pip_output=%%I"
-    goto :CheckModules
+:: Calling external python program to check for local modules
+python .\tools\check_local_modules.py
+if %errorlevel% equ 1 (
+    exit /b
 )
 
-:CheckModules
-REM Check if modules are found in the output
-if defined pip_output (
-    echo %yellow_text%=============================================================
-    echo Modules installed outside the virtual environment were found.
-    echo This can cause issues. Please review the installed modules.
-    echo.
-    echo You can uninstall all local modules with:
-    echo.
-    echo %blue_text%deactivate
-    echo pip freeze ^> uninstall.txt
-    echo pip uninstall -y -r uninstall.txt
-    echo %yellow_text%=============================================================%reset_text%
-    goto :PromptUser
-)
-
-:PromptUser
-REM Ask the user if they want to quit or continue
-echo.
-echo Do you want to:
-echo [1] - Quit
-echo [2] - Continue with the setup
-set /p "choice=Enter your choice (1 or 2): "
-if /i "%choice%"=="1" (
-    echo You chose to quit.
-    echo.
-    goto :Quit
-) else if /i "%choice%"=="2" (
-    echo You chose to continue.
-    echo.
-    goto :ContinueSetup
-) else (
-    echo Invalid choice. Please try again.
-    echo.
-    goto :PromptUser
-)
-
-:ContinueSetup
-endlocal
-
 call .\venv\Scripts\activate.bat
 
-REM Upgrade pip if needed
-python.exe -m pip install --upgrade pip
-
-echo Do you want to uninstall previous versions of torch and associated files before installing? Usefull if you are upgrading from torch 1.12.1 to torch 2.0.0 or if you are downgrading from torch 2.0.0 to torch 1.12.1.
-echo [1] - Yes
-echo [2] - No (recommanded for most)
-set /p uninstall_choice="Enter your choice (1 or 2): "
-
-if %uninstall_choice%==1 (
-    pip uninstall -y xformers torch torchvision triton
-)
+:: Upgrade pip if needed
+pip install --upgrade pip
 
+echo.
 echo Please choose the version of torch you want to install:
-echo [1] - v1 (torch 1.12.1) (Recommended)
-echo [2] - v2 (torch 2.0.0) (Experimental)
+echo [1] - v1 (torch 1.12.1) (Recommended for best compatibility)
+echo [2] - v2 (torch 2.0.0) (Experimental, faster but more prone to issues)
 set /p choice="Enter your choice (1 or 2): "
 
-:: Only does this section to cleanup the old custom dll versions that we used to use. No longer needed now with the new bitsandbytes version
-pip uninstall -y bitsandbytes
-IF EXIST ".\venv\Lib\site-packages\bitsandbytes" (
-    rmdir .\venv\Lib\site-packages\bitsandbytes
-)
-:::::::::::::::::::::::::
-
 if %choice%==1 (
     pip install torch==1.12.1+cu116 torchvision==0.13.1+cu116 --extra-index-url https://download.pytorch.org/whl/cu116
-    pip install --use-pep517 --upgrade -r requirements.txt
+    del ".\logs\status\torch_version" > nul 2>&1
+    echo 1 > ".\logs\status\torch_version"
+    pip install --use-pep517 --upgrade -r requirements_windows_torch1.txt
     pip install -U -I --no-deps https://github.com/C43H66N12O12S2/stable-diffusion-webui/releases/download/f/xformers-0.0.14.dev0-cp310-cp310-win_amd64.whl
 ) else (
     pip install torch==2.0.1+cu118 torchvision==0.15.2+cu118 --extra-index-url https://download.pytorch.org/whl/cu118
-    pip install --use-pep517 --upgrade -r requirements.txt
+    del ".\logs\status\torch_version" > nul 2>&1
+    echo 2 > ".\logs\status\torch_version"
+    pip install --use-pep517 --upgrade -r requirements_windows_torch2.txt
     pip install --upgrade xformers==0.0.20
-    rem pip install -U -I --no-deps https://files.pythonhosted.org/packages/d6/f7/02662286419a2652c899e2b3d1913c47723fc164b4ac06a85f769c291013/xformers-0.0.17rc482-cp310-cp310-win_amd64.whl
     pip install https://huggingface.co/r4ziel/xformers_pre_built/resolve/main/triton-2.0.0-cp310-cp310-win_amd64.whl
 )
 
 python.exe .\tools\update_bitsandbytes.py
 
-@REM copy /y .\bitsandbytes_windows\*.dll .\venv\Lib\site-packages\bitsandbytes\
-@REM copy /y .\bitsandbytes_windows\cextension.py .\venv\Lib\site-packages\bitsandbytes\cextension.py
-@REM copy /y .\bitsandbytes_windows\main.py .\venv\Lib\site-packages\bitsandbytes\cuda_setup\main.py
-
 accelerate config
-
-:Quit
-exit

test/img/10_darius kawasaki person/Dariusz_Zawadzki.txt

@@ -0,0 +1 @@
+a painting of a steam punk skull with a gas mask , by darius kawasaki

test/img/10_darius kawasaki person/Dariusz_Zawadzki_2.txt

@@ -0,0 +1 @@
+a painting of a man with a skull on his head , by darius kawasaki

test/img/10_darius kawasaki person/Dariusz_Zawadzki_3.txt

@@ -0,0 +1 @@
+a painting of a woman with a helmet on her head , by darius kawasaki

test/img/10_darius kawasaki person/Dariusz_Zawadzki_4.txt

@@ -0,0 +1 @@
+a painting of a horned man with a goat head , by darius kawasaki

test/img/10_darius kawasaki person/Dariusz_Zawadzki_5.txt

@@ -0,0 +1 @@
+a painting of a man playing a piano , by darius kawasaki