Refactor deleting old files in fetch cmd & improve performance

juraj-hrivnak · Aug 15, 2024 · 3a0fa57 · 3a0fa57
1 parent c2d9049
commit 3a0fa57
Show file tree

Hide file tree

Showing 4 changed files with 104 additions and 87 deletions.
diff --git a/src/commonMain/kotlin/teksturepako/pakku/api/actions/fetch/Fetch.kt b/src/commonMain/kotlin/teksturepako/pakku/api/actions/fetch/Fetch.kt
@@ -3,22 +3,23 @@ package teksturepako.pakku.api.actions.fetch
 import com.github.michaelbull.result.*
 import kotlinx.atomicfu.AtomicLong
 import kotlinx.atomicfu.atomic
-import kotlinx.coroutines.Dispatchers
-import kotlinx.coroutines.ExperimentalCoroutinesApi
+import kotlinx.coroutines.*
 import kotlinx.coroutines.channels.consumeEach
 import kotlinx.coroutines.channels.produce
-import kotlinx.coroutines.coroutineScope
-import kotlinx.coroutines.launch
 import teksturepako.pakku.api.actions.ActionError
 import teksturepako.pakku.api.actions.ActionError.*
+import teksturepako.pakku.api.actions.sync.getFileHashes
 import teksturepako.pakku.api.data.LockFile
+import teksturepako.pakku.api.data.workingPath
 import teksturepako.pakku.api.http.Http
+import teksturepako.pakku.api.overrides.ProjectOverride
 import teksturepako.pakku.api.platforms.Platform
 import teksturepako.pakku.api.projects.Project
 import teksturepako.pakku.api.projects.ProjectFile
-import kotlin.io.path.createParentDirectories
-import kotlin.io.path.exists
-import kotlin.io.path.writeBytes
+import teksturepako.pakku.api.projects.ProjectType
+import teksturepako.pakku.io.createHash
+import java.nio.file.Path
+import kotlin.io.path.*
 
 fun retrieveProjectFiles(
     lockFile: LockFile,
@@ -28,23 +29,19 @@ fun retrieveProjectFiles(
         project.getFilesForPlatform(platform).firstOrNull()
     }
 
-    if (file == null)
-    {
-        Err(NoFiles(project, lockFile))
-    }
-    else Ok(file)
+    if (file == null) Err(NoFiles(project, lockFile)) else Ok(file)
 }
 
 @OptIn(ExperimentalCoroutinesApi::class)
 suspend fun List<ProjectFile>.fetch(
     onError: suspend (error: ActionError) -> Unit,
-    onProgress: suspend (advance: Long, total: Long, sent: Long) -> Unit,
+    onProgress: suspend (advance: Long, total: Long) -> Unit,
     onSuccess: suspend (projectFile: ProjectFile, project: Project?) -> Unit,
     lockFile: LockFile,
 ) = coroutineScope {
     val maxBytes: AtomicLong = atomic(0L)
 
-    produce {
+    val channel = produce {
         for (projectFile in this@fetch)
         {
             launch {
@@ -58,7 +55,7 @@ suspend fun List<ProjectFile>.fetch(
                 val prevBytes: AtomicLong = atomic(0L)
 
                 val bytes = Http().requestByteArray(projectFile.url!!) { bytesSentTotal, _ ->
-                    onProgress(bytesSentTotal - prevBytes.value, maxBytes.value, bytesSentTotal)
+                    onProgress(bytesSentTotal - prevBytes.value, maxBytes.value)
                     prevBytes.getAndSet(bytesSentTotal)
                 }
 
@@ -76,7 +73,9 @@ suspend fun List<ProjectFile>.fetch(
                 send(projectFile to bytes)
             }
         }
-    }.consumeEach { (projectFile, bytes) ->
+    }
+
+    channel.consumeEach { (projectFile, bytes) ->
         launch(Dispatchers.IO) {
             runCatching {
                 val file = projectFile.getPath()
@@ -89,4 +88,52 @@ suspend fun List<ProjectFile>.fetch(
             }
         }
     }
+
+    launch {
+        if (channel.isEmpty) this.cancel()
+    }
+}
+
+@OptIn(ExperimentalCoroutinesApi::class)
+suspend fun deleteOldFiles(
+    onSuccess: suspend (file: Path) -> Unit,
+    projectFiles: List<ProjectFile>,
+    projectOverrides: Set<ProjectOverride>
+) = coroutineScope {
+    val projectFileNames = projectFiles.filter { it.hashes?.get("sha1") == null }.map { it.fileName }
+    val projectFileHashes = projectFiles.mapNotNull { projectFile -> projectFile.hashes?.get("sha1") }
+
+    val channel = produce {
+        ProjectType.entries
+            .filterNot { it == ProjectType.WORLD }
+            .mapNotNull { projectType ->
+                val folder = Path(workingPath, projectType.folderName)
+                if (folder.notExists()) return@mapNotNull null
+                runCatching { folder.listDirectoryEntries() }.get()
+            }.flatten().forEach { file ->
+                launch {
+                    val bytes = runCatching { file.readBytes() }.get()
+                    val fileHash = bytes?.let { createHash("sha1", it) }
+
+                    if (file.extension in listOf("jar", "zip")
+                        && fileHash !in projectOverrides.getFileHashes()
+                        && file.name !in projectFileNames
+                        && fileHash !in projectFileHashes)
+                    {
+                        send(file)
+                    }
+                }
+            }
+    }
+
+    channel.consumeEach { file ->
+        launch(Dispatchers.IO) {
+            val deleted = file.deleteIfExists()
+            if (deleted) onSuccess(file)
+        }
+    }
+
+    launch {
+        if (channel.isEmpty) this.cancel()
+    }
 }
diff --git a/src/commonMain/kotlin/teksturepako/pakku/api/platforms/CurseForge.kt b/src/commonMain/kotlin/teksturepako/pakku/api/platforms/CurseForge.kt
@@ -260,8 +260,6 @@ object CurseForge : Platform(
 
         val murmurs = bytes.map { it.toMurmur2() }
 
-        debug { println(murmurs) }
-
         val response = json.decodeFromString<GetFingerprintsMatchesResponse>(
             this.requestProjectBody("fingerprints/432", GetFingerprintsMatches(murmurs))
                 ?: return mutableSetOf()
@@ -287,8 +285,6 @@ object CurseForge : Platform(
 
         val murmurs = bytes.map { it.toMurmur2() }
 
-        debug { println(murmurs) }
-
         return json.decodeFromString<GetFingerprintsMatchesResponse>(
             this.requestProjectBody("fingerprints/432", GetFingerprintsMatches(murmurs))
                 ?: return mutableSetOf()

diff --git a/src/commonMain/kotlin/teksturepako/pakku/cli/cmd/Fetch.kt b/src/commonMain/kotlin/teksturepako/pakku/cli/cmd/Fetch.kt
@@ -4,29 +4,21 @@ import com.github.ajalt.clikt.core.CliktCommand
 import com.github.ajalt.clikt.core.terminal
 import com.github.ajalt.mordant.animation.progressAnimation
 import com.github.ajalt.mordant.widgets.Spinner
-import com.github.michaelbull.result.get
 import com.github.michaelbull.result.getOrElse
-import com.github.michaelbull.result.runCatching
-import kotlinx.coroutines.Dispatchers
-import kotlinx.coroutines.joinAll
 import kotlinx.coroutines.launch
 import kotlinx.coroutines.runBlocking
 import teksturepako.pakku.api.actions.ActionError.AlreadyExists
+import teksturepako.pakku.api.actions.fetch.deleteOldFiles
 import teksturepako.pakku.api.actions.fetch.fetch
 import teksturepako.pakku.api.actions.fetch.retrieveProjectFiles
-import teksturepako.pakku.api.actions.sync.getFileHashes
 import teksturepako.pakku.api.actions.sync.sync
 import teksturepako.pakku.api.data.LockFile
-import teksturepako.pakku.api.data.workingPath
 import teksturepako.pakku.api.overrides.readProjectOverrides
 import teksturepako.pakku.api.platforms.Multiplatform
-import teksturepako.pakku.api.projects.ProjectType
 import teksturepako.pakku.cli.ui.pDanger
 import teksturepako.pakku.cli.ui.pError
 import teksturepako.pakku.cli.ui.pInfo
 import teksturepako.pakku.cli.ui.pSuccess
-import teksturepako.pakku.io.createHash
-import kotlin.io.path.*
 
 class Fetch : CliktCommand("Fetch projects to your modpack folder")
 {
@@ -51,72 +43,56 @@ class Fetch : CliktCommand("Fetch projects to your modpack folder")
             }
         }
 
-        projectFiles.fetch(
-            onError = { error ->
-                if (error !is AlreadyExists) terminal.pError(error)
-            },
-            onProgress = { advance, total, sent ->
-                progressBar.advance(advance)
-                progressBar.updateTotal(total)
+        val fetchJob = launch {
+            projectFiles.fetch(
+                onError = { error ->
+                    if (error !is AlreadyExists) terminal.pError(error)
+                },
+                onProgress = { advance, total ->
+                    progressBar.advance(advance)
+                    progressBar.updateTotal(total)
+                },
+                onSuccess = { projectFile, _ ->
+                    terminal.pSuccess("${projectFile.getPath()} saved")
+                },
+                lockFile
+            )
+        }
 
-                if (sent >= total)
-                {
-                    progressBar.clear()
-                    echo()
-                }
-            },
-            onSuccess = { projectFile, _ ->
-                terminal.pSuccess("${projectFile.getPath()} saved")
-            },
-            lockFile
-        )
+        fetchJob.invokeOnCompletion {
+            progressBar.clear()
+        }
 
         // -- OVERRIDES --
 
         val projectOverrides = readProjectOverrides()
 
-        projectOverrides.sync(
-            onError = { error ->
-                if (error !is AlreadyExists) terminal.pError(error)
-            },
-            onSuccess = { projectOverride ->
-                terminal.pInfo("${projectOverride.fullOutputPath} synced")
-            }
-        ).joinAll()
+        val syncJob = launch {
+            projectOverrides.sync(
+                onError = { error ->
+                    if (error !is AlreadyExists) terminal.pError(error)
+                },
+                onSuccess = { projectOverride ->
+                    terminal.pInfo("${projectOverride.fullOutputPath} synced")
+                }
+            )
+        }
 
         // -- OLD FILES --
 
-        val oldFiles = ProjectType.entries
-            .filterNot { it == ProjectType.WORLD }
-            .mapNotNull { projectType ->
-                val folder = Path(workingPath, projectType.folderName)
-                if (folder.notExists()) return@mapNotNull null
-                runCatching { folder.listDirectoryEntries() }.get()
-            }.flatMap { entry ->
-                entry.filter { file ->
-                    val bytes = runCatching { file.readBytes() }.get()
-                    val fileHash = bytes?.let { createHash("sha1", it) }
+        val oldFilesJob = launch {
+            deleteOldFiles(
+                onSuccess = { file ->
+                    terminal.pDanger("$file deleted")
+                },
+                projectFiles, projectOverrides
+            )
+        }
 
-                    val projectFileNames = projectFiles.filter { projectFile ->
-                        projectFile.hashes?.get("sha1") == null
-                    }.map { projectFile ->
-                        projectFile.fileName
-                    }
+        fetchJob.join()
+        syncJob.join()
+        oldFilesJob.join()
 
-                    file.extension in listOf("jar", "zip")
-                            && fileHash !in projectOverrides.getFileHashes()
-                            && file.name !in projectFileNames
-                            && fileHash !in projectFiles.mapNotNull { projectFile ->
-                                projectFile.hashes?.get("sha1")
-                            }
-                }
-            }
-
-        oldFiles.map {
-            launch(Dispatchers.IO) {
-                it.deleteIfExists()
-                terminal.pDanger("$it deleted")
-            }
-        }.joinAll()
+        echo()
     }
 }
diff --git a/src/commonMain/kotlin/teksturepako/pakku/cli/cmd/Status.kt b/src/commonMain/kotlin/teksturepako/pakku/cli/cmd/Status.kt
@@ -2,8 +2,6 @@ package teksturepako.pakku.cli.cmd
 
 import com.github.ajalt.clikt.core.CliktCommand
 import com.github.ajalt.clikt.core.terminal
-import com.github.ajalt.clikt.parameters.options.flag
-import com.github.ajalt.clikt.parameters.options.option
 import com.github.ajalt.mordant.table.grid
 import kotlinx.coroutines.runBlocking
 import teksturepako.pakku.api.data.ConfigFile