Check if data structures are complete and valid at startup

- Fixes crawler stall bug if the queue list is empty at startup which was introduced by the new database consistency checks (#284). - Clears queue list before reinitializing it with a possible local Queuelist.json if the database location was changed in the settings view. - Checks if for each IBlog exist an IFiles if the database location was changed in the settings view.
johanneszab · Nov 7, 2018 · 30b8dc1 · 30b8dc1
1 parent 3e0cc97
commit 30b8dc1
Show file tree

Hide file tree

Showing 8 changed files with 56 additions and 11 deletions.
diff --git a/src/TumblThree/TumblThree.Applications/Controllers/CrawlerController.cs b/src/TumblThree/TumblThree.Applications/Controllers/CrawlerController.cs
@@ -152,7 +152,7 @@ private async Task Crawl()
             pauseCommand.RaiseCanExecuteChanged();
             stopCommand.RaiseCanExecuteChanged();
 
-            await crawlerService.DatabasesLoaded.Task;
+            await Task.WhenAll(crawlerService.LibraryLoaded.Task, crawlerService.DatabasesLoaded.Task);
 
             for (var i = 0; i < shellService.Settings.ConcurrentBlogs; i++)
             {

diff --git a/src/TumblThree/TumblThree.Applications/Controllers/ManagerController.cs b/src/TumblThree/TumblThree.Applications/Controllers/ManagerController.cs
@@ -43,6 +43,7 @@ internal class ManagerController
 
         private readonly AsyncDelegateCommand checkStatusCommand;
         private readonly DelegateCommand copyUrlCommand;
+        private readonly DelegateCommand checkIfDatabasesCompleteCommand;
         private readonly AsyncDelegateCommand addBlogCommand;
         private readonly DelegateCommand autoDownloadCommand;
         private readonly DelegateCommand enqueueSelectedCommand;
@@ -88,6 +89,7 @@ public ManagerController(IShellService shellService, ISelectionService selection
             enqueueSelectedCommand = new DelegateCommand(EnqueueSelected, CanEnqueueSelected);
             loadLibraryCommand = new AsyncDelegateCommand(LoadLibraryAsync, CanLoadLibrary);
             loadAllDatabasesCommand = new AsyncDelegateCommand(LoadAllDatabasesAsync, CanLoadAllDatbases);
+            checkIfDatabasesCompleteCommand = new DelegateCommand(CheckIfDatabasesComplete, CanCheckIfDatabasesComplete);
             listenClipboardCommand = new DelegateCommand(ListenClipboard);
             autoDownloadCommand = new DelegateCommand(EnqueueAutoDownload, CanEnqueueAutoDownload);
             showDetailsCommand = new DelegateCommand(ShowDetailsCommand);
@@ -113,6 +115,7 @@ public async Task Initialize()
             crawlerService.EnqueueSelectedCommand = enqueueSelectedCommand;
             crawlerService.LoadLibraryCommand = loadLibraryCommand;
             crawlerService.LoadAllDatabasesCommand = loadAllDatabasesCommand;
+            crawlerService.CheckIfDatabasesCompleteCommand = checkIfDatabasesCompleteCommand;
             crawlerService.AutoDownloadCommand = autoDownloadCommand;
             crawlerService.ListenClipboardCommand = listenClipboardCommand;
             crawlerService.PropertyChanged += CrawlerServicePropertyChanged;
@@ -128,6 +131,7 @@ public async Task Initialize()
             ManagerViewModel.QueueItems = QueueManager.Items;
             QueueManager.Items.CollectionChanged += QueueItemsCollectionChanged;
             ManagerViewModel.QueueItems.CollectionChanged += ManagerViewModel.QueueItemsCollectionChanged;
+            BlogManagerFinishedLoadingLibrary += OnBlogManagerFinishedLoadingLibrary;
             BlogManagerFinishedLoadingDatabases += OnBlogManagerFinishedLoadingDatabases;
 
             shellService.ContentView = ManagerViewModel.View;
@@ -144,6 +148,11 @@ public void Shutdown()
         {
         }
 
+        private void OnBlogManagerFinishedLoadingLibrary(object sender, EventArgs e)
+        {
+            crawlerService.LibraryLoaded.SetResult(true);
+        }
+
         private void OnBlogManagerFinishedLoadingDatabases(object sender, EventArgs e)
         {
             crawlerService.DatabasesLoaded.SetResult(true);
@@ -164,8 +173,9 @@ private void QueueItemsCollectionChanged(object sender, NotifyCollectionChangedE
         private async Task LoadDataBasesAsync()
         {
             // TODO: Methods have side effects!
+            // They remove blogs from the blog manager.
             await LoadLibraryAsync();
-            await LoadAllDatabasesAsync();        
+            await LoadAllDatabasesAsync();
             CheckIfDatabasesComplete();
             await CheckBlogsOnlineStatusAsync();
         }
@@ -359,6 +369,11 @@ private bool CanLoadAllDatbases()
             return !crawlerService.IsCrawl;
         }
 
+        private bool CanCheckIfDatabasesComplete()
+        {
+            return crawlerService.DatabasesLoaded.Task.GetAwaiter().IsCompleted && crawlerService.LibraryLoaded.Task.GetAwaiter().IsCompleted;
+        }
+
         private bool CanEnqueueSelected()
         {
             return ManagerViewModel.SelectedBlogFile != null && ManagerViewModel.SelectedBlogFile.Online;

diff --git a/src/TumblThree/TumblThree.Applications/Controllers/ModuleController.cs b/src/TumblThree/TumblThree.Applications/Controllers/ModuleController.cs
@@ -106,13 +106,14 @@ public void Initialize()
             ManagerController.QueueManager = queueManager;
             ManagerController.ManagerSettings = managerSettings;
             ManagerController.BlogManagerFinishedLoadingLibrary += OnBlogManagerFinishedLoadingLibrary;
-            Task managerControllerInit = ManagerController.Initialize();
             QueueController.QueueSettings = queueSettings;
             QueueController.QueueManager = queueManager;
-            QueueController.Initialize();
             DetailsController.QueueManager = queueManager;
-            DetailsController.Initialize();
             CrawlerController.QueueManager = queueManager;
+
+            Task managerControllerInit = ManagerController.Initialize();
+            QueueController.Initialize();
+            DetailsController.Initialize();
             CrawlerController.Initialize();
             cookieService.SetUriCookie(cookieList);
         }

diff --git a/src/TumblThree/TumblThree.Applications/Controllers/QueueController.cs b/src/TumblThree/TumblThree.Applications/Controllers/QueueController.cs
@@ -91,6 +91,7 @@ public void Run()
 
         public void LoadQueue()
         {
+            ClearList();
             IReadOnlyList<string> blogNamesToLoad = QueueSettings.Names;
             IReadOnlyList<BlogTypes> blogTypesToLoad = QueueSettings.Types;
             InsertFilesCore(0, blogNamesToLoad, blogTypesToLoad);

diff --git a/src/TumblThree/TumblThree.Applications/Services/CrawlerService.cs b/src/TumblThree/TumblThree.Applications/Services/CrawlerService.cs
@@ -23,9 +23,11 @@ public class CrawlerService : Model, ICrawlerService
         private ICommand enqueueSelectedCommand;
         private ICommand loadLibraryCommand;
         private ICommand loadAllDatabasesCommand;
+        private ICommand checkIfDatabasesCompleteCommand;
         private bool isCrawl;
         private bool isPaused;
         private bool isTimerSet;
+        private TaskCompletionSource<bool> libraryLoaded;
         private TaskCompletionSource<bool> databasesLoaded;
         private ICommand listenClipboardCommand;
         private string newBlogUrl;
@@ -47,6 +49,7 @@ public CrawlerService(IShellService shellService)
 
             activeItems = new ObservableCollection<QueueListItem>();
             readonlyActiveItems = new ReadOnlyObservableList<QueueListItem>(activeItems);
+            libraryLoaded = new TaskCompletionSource<bool>();
             databasesLoaded = new TaskCompletionSource<bool>();
             activeItems.CollectionChanged += ActiveItemsCollectionChanged;
         }
@@ -57,6 +60,12 @@ public bool IsTimerSet
             set => SetProperty(ref isTimerSet, value);
         }
 
+        public TaskCompletionSource<bool> LibraryLoaded
+        {
+            get => libraryLoaded;
+            set => SetProperty(ref libraryLoaded, value);
+        }
+
         public TaskCompletionSource<bool> DatabasesLoaded
         {
             get => databasesLoaded;
@@ -107,6 +116,12 @@ public ICommand LoadAllDatabasesCommand
             set => SetProperty(ref loadAllDatabasesCommand, value);
         }
 
+        public ICommand CheckIfDatabasesCompleteCommand
+        {
+            get => checkIfDatabasesCompleteCommand;
+            set => SetProperty(ref checkIfDatabasesCompleteCommand, value);
+        }
+
         public ICommand RemoveBlogFromQueueCommand
         {
             get => removeBlogFromQueueCommand;

diff --git a/src/TumblThree/TumblThree.Applications/Services/ICrawlerService.cs b/src/TumblThree/TumblThree.Applications/Services/ICrawlerService.cs
@@ -24,6 +24,8 @@ public interface ICrawlerService : INotifyPropertyChanged
 
         ICommand LoadAllDatabasesCommand { get; set; }
 
+        ICommand CheckIfDatabasesCompleteCommand { get; set; }
+
         ICommand RemoveBlogFromQueueCommand { get; set; }
 
         ICommand ListenClipboardCommand { get; set; }
@@ -52,6 +54,8 @@ public interface ICrawlerService : INotifyPropertyChanged
 
         Timer Timer { get; set; }
 
+        TaskCompletionSource<bool> LibraryLoaded { get; set; }
+
         TaskCompletionSource<bool> DatabasesLoaded { get; set; }
 
         void AddActiveItems(QueueListItem itemToAdd);

diff --git a/src/TumblThree/TumblThree.Applications/ViewModels/SettingsViewModel.cs b/src/TumblThree/TumblThree.Applications/ViewModels/SettingsViewModel.cs
@@ -27,7 +27,7 @@ public class SettingsViewModel : ViewModel<ISettingsView>
         private readonly DelegateCommand browseExportLocationCommand;
         private readonly DelegateCommand enableAutoDownloadCommand;
         private readonly DelegateCommand exportCommand;
-        private readonly DelegateCommand saveCommand;
+        private readonly AsyncDelegateCommand saveCommand;
         private readonly AsyncDelegateCommand tumblrLoginCommand;
         private readonly AsyncDelegateCommand tumblrLogoutCommand;
         private readonly AsyncDelegateCommand tumblrSubmitTFACommand;
@@ -143,7 +143,7 @@ public SettingsViewModel(ISettingsView view, IShellService shellService, ICrawle
             tumblrLoginCommand = new AsyncDelegateCommand(TumblrLogin);
             tumblrLogoutCommand = new AsyncDelegateCommand(TumblrLogout);
             tumblrSubmitTFACommand = new AsyncDelegateCommand(TumblrSubmitTFA);
-            saveCommand = new DelegateCommand(Save);
+            saveCommand = new AsyncDelegateCommand(Save);
             enableAutoDownloadCommand = new DelegateCommand(EnableAutoDownload);
             exportCommand = new DelegateCommand(ExportBlogs);
             bloglistExportFileType = new FileType(Resources.Textfile, SupportedFileTypes.BloglistExportFileType);
@@ -993,33 +993,39 @@ private void LoadSettings()
             }
         }
 
-        private void Save()
+        private async Task Save()
         {
             bool downloadLocationChanged = DownloadLocationChanged();
             bool loadAllDatabasesChanged = LoadAllDatabasesChanged();
             SaveSettings();
-            ApplySettings(downloadLocationChanged, loadAllDatabasesChanged);
+            await ApplySettings(downloadLocationChanged, loadAllDatabasesChanged);
         }
 
-        private void ApplySettings(bool downloadLocationChanged, bool loadAllDatabasesChanged)
+        private async Task ApplySettings(bool downloadLocationChanged, bool loadAllDatabasesChanged)
         {
             CrawlerService.Timeconstraint.SetRate((MaxConnections / (double)ConnectionTimeInterval));
 
             if (loadAllDatabasesChanged && downloadLocationChanged)
             {
+                CrawlerService.LibraryLoaded = new TaskCompletionSource<bool>();
                 CrawlerService.DatabasesLoaded = new TaskCompletionSource<bool>();
                 if (CrawlerService.StopCommand.CanExecute(null))
                     CrawlerService.StopCommand.Execute(null);
                 CrawlerService.LoadLibraryCommand.Execute(null);
                 CrawlerService.LoadAllDatabasesCommand.Execute(null);
+                await Task.WhenAll(CrawlerService.LibraryLoaded.Task, CrawlerService.DatabasesLoaded.Task);
+                CrawlerService.CheckIfDatabasesCompleteCommand.Execute(null);
             }
             else if (downloadLocationChanged)
             {
+                CrawlerService.LibraryLoaded = new TaskCompletionSource<bool>();
                 CrawlerService.DatabasesLoaded = new TaskCompletionSource<bool>();
                 if (CrawlerService.StopCommand.CanExecute(null))
                     CrawlerService.StopCommand.Execute(null);
                 CrawlerService.LoadLibraryCommand.Execute(null);
                 CrawlerService.LoadAllDatabasesCommand.Execute(null);
+                await Task.WhenAll(CrawlerService.LibraryLoaded.Task, CrawlerService.DatabasesLoaded.Task);
+                CrawlerService.CheckIfDatabasesCompleteCommand.Execute(null);
             }
             else if (loadAllDatabasesChanged)
             {

diff --git a/src/TumblThree/TumblThree.Presentation/DesignData/MockCrawlerService.cs b/src/TumblThree/TumblThree.Presentation/DesignData/MockCrawlerService.cs
@@ -50,6 +50,8 @@ public void RemoveActiveItem(QueueListItem itemToRemove)
 
         public ICommand LoadAllDatabasesCommand { get; set; }
 
+        public ICommand CheckIfDatabasesCompleteCommand { get; set; }
+
         public ICommand ListenClipboardCommand { get; set; }
 
         public ICommand CrawlCommand { get; set; }
@@ -75,8 +77,9 @@ public void RemoveActiveItem(QueueListItem itemToRemove)
 
         public Timer Timer { get; set; }
 
-        public TaskCompletionSource<bool> DatabasesLoaded { get; set; }
+        public TaskCompletionSource<bool> LibraryLoaded { get; set; }
 
+        public TaskCompletionSource<bool> DatabasesLoaded { get; set; }
 
         public void SetActiveBlogFiles(IEnumerable<IBlog> blogFilesToAdd)
         {