Refresh folder list after modifying watched Folders

Add dialog to scan and remove or mark for deletion of unavailable images
RupertAvery · Apr 20, 2024 · 0772a39 · 0772a39
1 parent 9f6498e
commit 0772a39
Show file tree

Hide file tree

Showing 20 changed files with 490 additions and 23 deletions.
diff --git a/Diffusion.Database/DataStore.Image.cs b/Diffusion.Database/DataStore.Image.cs
@@ -99,6 +99,17 @@ public IEnumerable<ImagePath> GetAllPathImages(string path)
             db.Close();
         }
 
+        public int CountAllPathImages(string path)
+        {
+            using var db = OpenConnection();
+
+            var count = db.ExecuteScalar<int>("SELECT COUNT(*) FROM Image WHERE PATH LIKE ? || '%'", path);
+
+            db.Close();
+
+            return count;
+        }
+
         public IEnumerable<ImagePath> GetFolderImages(int folderId)
         {
             using var db = OpenConnection();

diff --git a/Diffusion.Toolkit/Converters/UnavailableConverter.cs b/Diffusion.Toolkit/Converters/UnavailableConverter.cs
@@ -0,0 +1,19 @@
+using System;
+using System.Globalization;
+using System.Windows;
+using System.Windows.Data;
+
+namespace Diffusion.Toolkit.Converters;
+
+public class UnavailableConverter : IValueConverter
+{
+    public object Convert(object value, Type targetType, object parameter, CultureInfo culture)
+    {
+        return (bool)value ? TextDecorations.Strikethrough : null;
+    }
+
+    public object ConvertBack(object value, Type targetType, object parameter, CultureInfo culture)
+    {
+        throw new NotImplementedException();
+    }
+}
diff --git a/Diffusion.Toolkit/Localization/de-DE.json b/Diffusion.Toolkit/Localization/de-DE.json
@@ -31,7 +31,6 @@
   "Menu.Tools.SearchResults.RemoveMatching": "_Alle Suchergebnisse aus der Datenbank entfernen",
   "Menu.Tools.AutoTagNSFW": "_NSFW automatisch taggen",
   "Menu.Tools.Folders": "_Ordner",
-  "Menu.Tools.Folders.": "_Ordner",
   "Menu.Tools.Folders.FixMissingImages": "_Fehlende Bilder reparieren",
   "Menu.Tools.Folders.RemoveExcludedImages": "_Ausgeschlossene Bilder entfernen",
   "Menu.Tools.Folders.CleanRemovedFolders": "_Entfernte Ordner bereinigen",

diff --git a/Diffusion.Toolkit/Localization/default.json b/Diffusion.Toolkit/Localization/default.json
@@ -32,10 +32,10 @@
   "Menu.Tools.SearchResults.RemoveMatching": "_Remove all query results from the database",
   "Menu.Tools.AutoTagNSFW": "_Auto tag NSFW",
   "Menu.Tools.Folders": "_Folders",
-  "Menu.Tools.Folders.": "_Folders",
   "Menu.Tools.Folders.FixMissingImages": "_Fix missing images",
   "Menu.Tools.Folders.RemoveExcludedImages": "_Remove excluded images",
   "Menu.Tools.Folders.CleanRemovedFolders": "_Clean removed folders",
+  "Menu.Tools.UnavailableFiles": "Unavailable Files",
   "Menu.Help": "_Help",
   "Menu.Help.About": "About",
   "Menu.Help.GettingStarted": "Getting Started",
@@ -47,6 +47,7 @@
   "Common.Buttons.Browse": "Browse",
   "Common.Buttons.Apply": "Apply",
   "Common.Buttons.Close": "Close",
+  "Common.Buttons.Start": "Start",
   "Common.MessageBox.Cancel": "Confirm Cancel",
   "Common.MessageBox.ConfirmCancelOperation": "Are you sure you want to cancel the operation?",
   "Common.MessageBox.OperationCancelled": "The operation was cancelled.",
@@ -118,6 +119,7 @@
   "Search.Buttons.ShowFilter.ToolTip": "Show filter",
   "Search.Buttons.Clear.ToolTip": "Clear",
   "Search.Buttons.Refresh.ToolTip": "Refresh",
+  "Search.Folders.Buttons.Refresh.ToopTip": "Refresh",
   "Search.Albums.Buttons.SortAlbums.ToolTip": "Sort",
   "Search.Albums.Buttons.CreateAlbum.ToolTip": "New Album",
   "Search.AddressBar.Buttons.Home.ToolTip": "Home",
@@ -193,5 +195,15 @@
   "Messages.Captions.Error": "Error",
   "Messages.Errors.NoImagePaths": "No image paths configured!",
   "Main.Status.ImagesInDatabase": "{count} images in database",
-  "Main.Update.UpdateAvailable": "An update is available. Do you want to install now?"
+  "Main.Update.UpdateAvailable": "An update is available. Do you want to install now?",
+  "UnavailableFiles": "Scan for unavailable files",
+  "UnavailableFiles.Description": "This will scan for unavailable files in the specified locations. You can choose to immediately delete or mark for deletion",
+  "UnavailableFiles.Sources.SourceLabel": "Scan selected locations:",
+  "UnavailableFiles.Sources.RootFolders": "Root Folders",
+  "UnavailableFiles.Sources.SpecificFolders": "Specific Folders",
+  "UnavailableFiles.Options.RemoveFromUnavailableRootFolders": "Remove files from unavailable root folders",
+  "UnavailableFiles.Options.DeleteImmediately": "Delete immediately",
+  "UnavailableFiles.Options.MarkForDeletion": "Mark for deletion",
+  "UnavailableFiles.Results.Removed": "{count} images removed.",
+  "UnavailableFiles.Results.MarkedForDeletion": "{count} images marked for deletion."
 }
diff --git a/Diffusion.Toolkit/Localization/en-US.json b/Diffusion.Toolkit/Localization/en-US.json
@@ -31,7 +31,6 @@
   "Menu.Tools.SearchResults.RemoveMatching": "_Remove all query results from the database",
   "Menu.Tools.AutoTagNSFW": "_Auto tag NSFW",
   "Menu.Tools.Folders": "_Folders",
-  "Menu.Tools.Folders.": "_Folders",
   "Menu.Tools.Folders.FixMissingImages": "_Fix missing images",
   "Menu.Tools.Folders.RemoveExcludedImages": "_Remove excluded images",
   "Menu.Tools.Folders.CleanRemovedFolders": "_Clean removed folders",

diff --git a/Diffusion.Toolkit/Localization/es-ES.json b/Diffusion.Toolkit/Localization/es-ES.json
@@ -31,7 +31,6 @@
   "Menu.Tools.SearchResults.RemoveMatching": "Eliminar todos los resultados de la consulta de la base de datos",
   "Menu.Tools.AutoTagNSFW": "Etiquetar automáticamente _NSFW",
   "Menu.Tools.Folders": "_Carpetas",
-  "Menu.Tools.Folders.": "_Carpetas",
   "Menu.Tools.Folders.FixMissingImages": "_Corregir imágenes faltantes",
   "Menu.Tools.Folders.RemoveExcludedImages": "_Eliminar imágenes excluidas",
   "Menu.Tools.Folders.CleanRemovedFolders": "_Limpiar carpetas eliminadas",

diff --git a/Diffusion.Toolkit/Localization/fr-FR.json b/Diffusion.Toolkit/Localization/fr-FR.json
@@ -31,7 +31,6 @@
   "Menu.Tools.SearchResults.RemoveMatching": "_Supprimer tous les résultats de la requête de la base de données",
   "Menu.Tools.AutoTagNSFW": "_Baliser automatiquement comme NSFW",
   "Menu.Tools.Folders": "_Dossiers",
-  "Menu.Tools.Folders.": "_Dossiers",
   "Menu.Tools.Folders.FixMissingImages": "_Réparer les images manquantes",
   "Menu.Tools.Folders.RemoveExcludedImages": "_Supprimer les images exclues",
   "Menu.Tools.Folders.CleanRemovedFolders": "_Nettoyer les dossiers supprimés",

diff --git a/Diffusion.Toolkit/Localization/ja-JP.json b/Diffusion.Toolkit/Localization/ja-JP.json
@@ -31,7 +31,6 @@
   "Menu.Tools.SearchResults.RemoveMatching": "データベースからすべてのクエリ結果を削除",
   "Menu.Tools.AutoTagNSFW": "NSFWを自動タグ付け",
   "Menu.Tools.Folders": "フォルダ",
-  "Menu.Tools.Folders.": "フォルダ",
   "Menu.Tools.Folders.FixMissingImages": "不足している画像を修正",
   "Menu.Tools.Folders.RemoveExcludedImages": "除外された画像を削除",
   "Menu.Tools.Folders.CleanRemovedFolders": "削除されたフォルダをクリーンアップ",

diff --git a/Diffusion.Toolkit/MainWindow.xaml b/Diffusion.Toolkit/MainWindow.xaml
@@ -119,6 +119,7 @@
                     <MenuItem Header="{lex:Loc Menu.Tools.Folders.RemoveExcludedImages}" Command="{Binding RemoveExcludedImagesCommand}"/>
                     <MenuItem Header="{lex:Loc Menu.Tools.Folders.CleanRemovedFolders}" Command="{Binding CleanRemovedFoldersCommand}"/>
                 </MenuItem>
+                <MenuItem Header="{lex:Loc Menu.Tools.UnavailableFiles}" Command="{Binding UnavailableFilesCommand}"></MenuItem>
             </MenuItem>
             <MenuItem Header="{lex:Loc Menu.Help}">
                 <MenuItem Header="{lex:Loc Menu.Help.About}" Command="{Binding AboutCommand}"/>

diff --git a/Diffusion.Toolkit/MainWindow.xaml.Folders.cs b/Diffusion.Toolkit/MainWindow.xaml.Folders.cs
@@ -3,18 +3,9 @@
 using System.Collections.ObjectModel;
 using System.IO;
 using System.Linq;
-using System.Text;
-using System.Text.Json.Serialization;
-using System.Text.Json;
 using System.Threading.Tasks;
-using Diffusion.Civitai;
-using Diffusion.Civitai.Models;
-using System.Diagnostics;
-using System.Threading;
 using Diffusion.Toolkit.Classes;
 using Diffusion.Toolkit.Models;
-using static System.Net.Mime.MediaTypeNames;
-using System.Windows.Shapes;
 using SQLite;
 using Path = System.IO.Path;
 
@@ -290,7 +281,8 @@ private void LoadFolders()
                 Visible = true,
                 Depth = 0,
                 Name = path,
-                Path = path
+                Path = path,
+                IsUnavailable = !Directory.Exists(path)
             }));
         }
     }

diff --git a/Diffusion.Toolkit/MainWindow.xaml.Scanning.cs b/Diffusion.Toolkit/MainWindow.xaml.Scanning.cs
@@ -14,6 +14,165 @@ namespace Diffusion.Toolkit
 {
     public partial class MainWindow
     {
+        private Task ScanUnavailable(UnavailableFilesModel options)
+        {
+            _progressCancellationTokenSource = new CancellationTokenSource();
+
+            var token = _progressCancellationTokenSource.Token;
+
+            return Task.Run(() =>
+            {
+                var candidateImages = new List<int>();
+
+                Dispatcher.Invoke(() =>
+                {
+                    _model.IsBusy = true;
+                });
+
+                if (options.UseRootFolders)
+                {
+                    var rootFolders = options.ImagePaths.Where(f => f.IsSelected);
+
+                    var total = 0;
+
+                    foreach (var folder in rootFolders)
+                    {
+                        total += _dataStore.CountAllPathImages(folder.Path);
+                    }
+
+                    Dispatcher.Invoke(() =>
+                    {
+                        _model.CurrentProgress = 0;
+                        _model.TotalProgress = total;
+                    });
+
+                    var current = 0;
+
+                    var scanning = GetLocalizedText("Actions.Scanning.Status");
+
+                    foreach (var folder in rootFolders)
+                    {
+                        if (token.IsCancellationRequested)
+                        {
+                            break;
+                        }
+
+                        HashSet<string> ignoreFiles = new HashSet<string>();
+
+                        var folderImages = _dataStore.GetAllPathImages(folder.Path).ToDictionary(f => f.Path);
+
+                        if (Directory.Exists(folder.Path))
+                        {
+                            //var filesOnDisk = MetadataScanner.GetFiles(folder.Path, _settings.FileExtensions, null, _settings.RecurseFolders.GetValueOrDefault(true), null);
+                            var filesOnDisk = MetadataScanner.GetFiles(folder.Path, _settings.FileExtensions, ignoreFiles, _settings.RecurseFolders.GetValueOrDefault(true), _settings.ExcludePaths);
+
+                            foreach (var file in filesOnDisk)
+                            {
+                                if (token.IsCancellationRequested)
+                                {
+                                    break;
+                                }
+
+                                if (folderImages.ContainsKey(file))
+                                {
+                                    folderImages.Remove(file);
+                                }
+
+                                current++;
+
+                                if (current % 113 == 0)
+                                {
+
+                                    Dispatcher.Invoke(() =>
+                                    {
+                                        _model.CurrentProgress = current;
+
+                                        var status = scanning
+                                            .Replace("{current}", $"{_model.CurrentProgress:#,###,##0}")
+                                            .Replace("{total}", $"{_model.TotalProgress:#,###,##0}");
+                                        
+                                        _model.Status = status;
+                                    });
+                                }
+                            }
+
+                            foreach (var folderImage in folderImages)
+                            {
+                                candidateImages.Add(folderImage.Value.Id);
+                            }
+                        }
+                        else
+                        {
+                            if (options.RemoveFromUnavailableRootFolders)
+                            {
+                                foreach (var folderImage in folderImages)
+                                {
+                                    candidateImages.Add(folderImage.Value.Id);
+
+                                    current++;
+
+                                    if (current % 113 == 0)
+                                    {
+                                        Dispatcher.Invoke(() =>
+                                        {
+                                            _model.CurrentProgress = current;
+
+                                            var status = scanning
+                                                .Replace("{current}", $"{_model.CurrentProgress:#,###,##0}")
+                                                .Replace("{total}", $"{_model.TotalProgress:#,###,##0}");
+
+                                            _model.Status = status;
+                                        });
+                                    }
+                                }
+                            }
+
+                        }
+                    }
+
+                    Dispatcher.Invoke(() =>
+                    {
+                        _model.CurrentProgress = total;
+                        _model.TotalProgress = total;
+
+                        var unavailableFiles = GetLocalizedText("UnavailableFiles");
+
+                        if (options.DeleteImmediately)
+                        {
+                            _dataStore.RemoveImages(candidateImages);
+                            
+                            var removed = GetLocalizedText("UnavailableFiles.Results.Removed");
+
+                            removed = removed.Replace("{count}", $"{candidateImages.Count:#,###,##0}");
+
+                            _messagePopupManager.Show(removed, unavailableFiles, PopupButtons.OK);
+                        }
+                        else
+                        {
+                            _dataStore.SetDeleted(candidateImages, true);
+
+                            var marked = GetLocalizedText("UnavailableFiles.Results.MarkedForDeletion");
+
+                            marked = marked.Replace("{count}", $"{candidateImages.Count:#,###,##0}");
+
+                            _messagePopupManager.Show(marked, unavailableFiles, PopupButtons.OK);
+                        }
+
+                        var completed = GetLocalizedText("Actions.Scanning.Completed");
+
+                        _model.IsBusy = true;
+                        _model.Status = completed;
+                        _model.CurrentProgress = 0;
+                        _model.TotalProgress = Int32.MaxValue;
+                    });
+
+                }
+
+
+            });
+
+        }
+
 
         private async Task RescanTask(object o)
         {

diff --git a/Diffusion.Toolkit/MainWindow.xaml.cs b/Diffusion.Toolkit/MainWindow.xaml.cs
@@ -136,6 +136,8 @@ public MainWindow()
             _model.RemoveExcludedImagesCommand = new RelayCommand<object>((o) => CleanExcludedPaths());
             _model.CleanRemovedFoldersCommand = new AsyncCommand<object>(CleanRemovedFolders);
 
+            _model.UnavailableFilesCommand = new RelayCommand<object>((o) => UnavailableFiles());
+
             _model.ShowFilterCommand = new RelayCommand<object>((o) => _search?.ShowFilter());
             _model.ToggleAutoRefresh = new RelayCommand<object>((o) => ToggleAutoRefresh());
 
@@ -188,6 +190,17 @@ public MainWindow()
             //System.Diagnostics.Debug.Write(str);
         }
 
+        private void UnavailableFiles()
+        {
+            var window = new UnavailableFilesWindow(_dataStore, _settings);
+            window.Owner = this;
+            window.ShowDialog();
+            if (window.DialogResult is true)
+            {
+                ScanUnavailable(window.Model);
+            }
+        }
+
         private void ToggleNavigationPane()
         {
             _model.Settings.NavigationSection.ToggleSection();
@@ -838,6 +851,8 @@ private async void ShowSettings(object obj)
                             CleanRemovedFoldersInternal();
                         }
 
+                        LoadFolders();
+
                         await TryScanFolders();
 
                         // Rebuild watchers in case paths were added or removed

diff --git a/Diffusion.Toolkit/Models/FolderViewModel.cs b/Diffusion.Toolkit/Models/FolderViewModel.cs
@@ -17,6 +17,7 @@ public class FolderViewModel : BaseNotify
     private bool _visible;
     private bool _isSelected;
     private string _name;
+    private bool _isUnavailable;
 
     public FolderState State
     {
@@ -55,6 +56,12 @@ public string Name
         set => SetField(ref _name, value);
     }
 
+    public bool IsUnavailable
+    {
+        get => _isUnavailable;
+        set => SetField(ref _isUnavailable, value);
+    }
+
     public override bool Equals(object? obj)
     {
         if (obj is FolderViewModel folder)