Fix garbage collection of files in process store

Rewrite the stage deleting files to use Task Parallel Library instead of ParallelEnumerable.AsParallel. After observing the earlier implementation using `AsParallel` hanging.
pine-vm · Sep 29, 2022 · 81e61aa · 81e61aa
1 parent 55249f8
commit 81e61aa
Show file tree

Hide file tree

Showing 3 changed files with 22 additions and 9 deletions.
diff --git a/implement/elm-fullstack/Program.cs b/implement/elm-fullstack/Program.cs
@@ -15,7 +15,7 @@ namespace ElmFullstack;
 
 public class Program
 {
-    static public string AppVersionId => "2022-09-27";
+    static public string AppVersionId => "2022-09-29";
 
     static int AdminInterfaceDefaultPort => 4000;
 

diff --git a/implement/elm-fullstack/WebHost/StartupAdminInterface.cs b/implement/elm-fullstack/WebHost/StartupAdminInterface.cs
@@ -598,6 +598,10 @@ TruncateProcessHistoryReport truncateProcessHistory(TimeSpan productionBlockDura
                         .Select(g => g.Select(x => x.s).ToImmutableList())
                         .ToImmutableList();
 
+                    logger.LogInformation(
+                        message: nameof(truncateProcessHistory) + ": Found {filePathCount} file paths to delete",
+                        filePathsInProcessStorePartitions.Sum(partition => partition.Count));
+
                     lock (avoidConcurrencyLock)
                     {
                         var lockStopwatch = System.Diagnostics.Stopwatch.StartNew();
@@ -609,6 +613,10 @@ TruncateProcessHistoryReport truncateProcessHistory(TimeSpan productionBlockDura
 
                         storeReductionStopwatch.Stop();
 
+                        logger.LogInformation(
+                            message: nameof(truncateProcessHistory) + ": Stored reduction in {storeReductionDurationMs} ms",
+                            storeReductionStopwatch.ElapsedMilliseconds);
+
                         var getFilesForRestoreStopwatch = System.Diagnostics.Stopwatch.StartNew();
 
                         var filesForRestore =
@@ -621,11 +629,9 @@ TruncateProcessHistoryReport truncateProcessHistory(TimeSpan productionBlockDura
 
                         var deleteFilesStopwatch = System.Diagnostics.Stopwatch.StartNew();
 
-                        var totalDeletedFilesCount =
+                        var partitionsTasks =
                             filePathsInProcessStorePartitions
-                            .AsParallel()
-                            .WithDegreeOfParallelism(numbersOfThreadsToDeleteFiles)
-                            .Select(partitionFilePaths =>
+                            .Select(partitionFilePaths => System.Threading.Tasks.Task.Run(() =>
                             {
                                 int partitionDeletedFilesCount = 0;
 
@@ -642,11 +648,18 @@ TruncateProcessHistoryReport truncateProcessHistory(TimeSpan productionBlockDura
                                 }
 
                                 return partitionDeletedFilesCount;
-                            })
-                            .Sum();
+                            }))
+                            .ToImmutableList();
+
+                        var totalDeletedFilesCount = partitionsTasks.Sum(task => task.Result);
 
                         deleteFilesStopwatch.Stop();
 
+                        logger.LogInformation(
+                            message: nameof(truncateProcessHistory) + ": Deleted {totalDeletedFilesCount} files in {storeReductionDurationMs} ms",
+                            totalDeletedFilesCount,
+                            deleteFilesStopwatch.ElapsedMilliseconds);
+
                         return new TruncateProcessHistoryReport
                         (
                             beginTime: beginTime,

diff --git a/implement/elm-fullstack/elm-fullstack.csproj b/implement/elm-fullstack/elm-fullstack.csproj
@@ -5,8 +5,8 @@
     <TargetFramework>net7.0</TargetFramework>
     <RootNamespace>ElmFullstack</RootNamespace>
     <AssemblyName>elm-fs</AssemblyName>
-    <AssemblyVersion>2022.0927.0.0</AssemblyVersion>
-    <FileVersion>2022.0927.0.0</FileVersion>
+    <AssemblyVersion>2022.0929.0.0</AssemblyVersion>
+    <FileVersion>2022.0929.0.0</FileVersion>
     <Nullable>enable</Nullable>
   </PropertyGroup>