From 81e61aaada9bf95ea16b5cb75c09cd2e5d0b3743 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Michael=20R=C3=A4tzel?= <viir@viir.de>
Date: Thu, 29 Sep 2022 10:32:57 +0000
Subject: [PATCH] Fix garbage collection of files in process store

Rewrite the stage deleting files to use Task Parallel Library instead of ParallelEnumerable.AsParallel.
After observing the earlier implementation using `AsParallel` hanging.
---
 implement/elm-fullstack/Program.cs            |  2 +-
 .../WebHost/StartupAdminInterface.cs          | 25 ++++++++++++++-----
 implement/elm-fullstack/elm-fullstack.csproj  |  4 +--
 3 files changed, 22 insertions(+), 9 deletions(-)

diff --git a/implement/elm-fullstack/Program.cs b/implement/elm-fullstack/Program.cs
index 5eed89db..c314252c 100644
--- a/implement/elm-fullstack/Program.cs
+++ b/implement/elm-fullstack/Program.cs
@@ -15,7 +15,7 @@ namespace ElmFullstack;
 
 public class Program
 {
-    static public string AppVersionId => "2022-09-27";
+    static public string AppVersionId => "2022-09-29";
 
     static int AdminInterfaceDefaultPort => 4000;
 
diff --git a/implement/elm-fullstack/WebHost/StartupAdminInterface.cs b/implement/elm-fullstack/WebHost/StartupAdminInterface.cs
index 296f99e1..549e761b 100644
--- a/implement/elm-fullstack/WebHost/StartupAdminInterface.cs
+++ b/implement/elm-fullstack/WebHost/StartupAdminInterface.cs
@@ -598,6 +598,10 @@ TruncateProcessHistoryReport truncateProcessHistory(TimeSpan productionBlockDura
                         .Select(g => g.Select(x => x.s).ToImmutableList())
                         .ToImmutableList();
 
+                    logger.LogInformation(
+                        message: nameof(truncateProcessHistory) + ": Found {filePathCount} file paths to delete",
+                        filePathsInProcessStorePartitions.Sum(partition => partition.Count));
+
                     lock (avoidConcurrencyLock)
                     {
                         var lockStopwatch = System.Diagnostics.Stopwatch.StartNew();
@@ -609,6 +613,10 @@ TruncateProcessHistoryReport truncateProcessHistory(TimeSpan productionBlockDura
 
                         storeReductionStopwatch.Stop();
 
+                        logger.LogInformation(
+                            message: nameof(truncateProcessHistory) + ": Stored reduction in {storeReductionDurationMs} ms",
+                            storeReductionStopwatch.ElapsedMilliseconds);
+
                         var getFilesForRestoreStopwatch = System.Diagnostics.Stopwatch.StartNew();
 
                         var filesForRestore =
@@ -621,11 +629,9 @@ TruncateProcessHistoryReport truncateProcessHistory(TimeSpan productionBlockDura
 
                         var deleteFilesStopwatch = System.Diagnostics.Stopwatch.StartNew();
 
-                        var totalDeletedFilesCount =
+                        var partitionsTasks =
                             filePathsInProcessStorePartitions
-                            .AsParallel()
-                            .WithDegreeOfParallelism(numbersOfThreadsToDeleteFiles)
-                            .Select(partitionFilePaths =>
+                            .Select(partitionFilePaths => System.Threading.Tasks.Task.Run(() =>
                             {
                                 int partitionDeletedFilesCount = 0;
 
@@ -642,11 +648,18 @@ TruncateProcessHistoryReport truncateProcessHistory(TimeSpan productionBlockDura
                                 }
 
                                 return partitionDeletedFilesCount;
-                            })
-                            .Sum();
+                            }))
+                            .ToImmutableList();
+
+                        var totalDeletedFilesCount = partitionsTasks.Sum(task => task.Result);
 
                         deleteFilesStopwatch.Stop();
 
+                        logger.LogInformation(
+                            message: nameof(truncateProcessHistory) + ": Deleted {totalDeletedFilesCount} files in {storeReductionDurationMs} ms",
+                            totalDeletedFilesCount,
+                            deleteFilesStopwatch.ElapsedMilliseconds);
+
                         return new TruncateProcessHistoryReport
                         (
                             beginTime: beginTime,
diff --git a/implement/elm-fullstack/elm-fullstack.csproj b/implement/elm-fullstack/elm-fullstack.csproj
index 54336d8e..d2715388 100644
--- a/implement/elm-fullstack/elm-fullstack.csproj
+++ b/implement/elm-fullstack/elm-fullstack.csproj
@@ -5,8 +5,8 @@
     <TargetFramework>net7.0</TargetFramework>
     <RootNamespace>ElmFullstack</RootNamespace>
     <AssemblyName>elm-fs</AssemblyName>
-    <AssemblyVersion>2022.0927.0.0</AssemblyVersion>
-    <FileVersion>2022.0927.0.0</FileVersion>
+    <AssemblyVersion>2022.0929.0.0</AssemblyVersion>
+    <FileVersion>2022.0929.0.0</FileVersion>
     <Nullable>enable</Nullable>
   </PropertyGroup>