Optimize source import

assets/src/main/kotlin/org/wycliffeassociates/otter/assets/initialization/InitializeProjects.kt

@@ -48,7 +48,7 @@ class InitializeProjects @Inject constructor(
     private val rcImporterFactory: RCImporterFactory
 ) : Installable {
     override val name = "PROJECTS"
-    override val version = 1
+    override val version = 2
 
     private val log = LoggerFactory.getLogger(InitializeProjects::class.java)
     private lateinit var callback: ProjectImporterCallback
@@ -80,10 +80,10 @@ class InitializeProjects @Inject constructor(
 
     private fun migrate() {
         `migrate to version 1`()
+        `migrate sources to version 2`()
     }
 
     private fun `migrate to version 1`() {
-        `migrate sources to version 1`()
         `migrate takes to version 1`()
 
         val projects = fetchProjects()
@@ -119,27 +119,35 @@ class InitializeProjects @Inject constructor(
         projectFilesAccessor.writeSelectedTakesFile(workbook, projectIsBook)
     }
 
-    private fun `migrate sources to version 1`() {
+    private fun `migrate sources to version 2`() {
         resourceMetadataRepo.getAllSources().blockingGet()
             .forEach { resourceMetadata ->
-                if (resourceMetadata.path.isDirectory) {
-                    // Compress resource container
-                    val zipName = "${resourceMetadata.language.slug}_${resourceMetadata.identifier}"
-                    val tempZip = createTempFile(zipName, "zip")
-                    tempZip.parentFile.deleteOnExit()
-
-                    directoryProvider.newFileWriter(tempZip).use { fileWriter ->
-                        fileWriter.copyDirectory(resourceMetadata.path, "/")
+                if (resourceMetadata.path.isFile) {
+                    val sourceFile = resourceMetadata.path
+                    val dirName = "${resourceMetadata.language.slug}_${resourceMetadata.identifier}-source"
+                    val targetDir = sourceFile.parentFile.resolve(dirName)
+                    if (targetDir.exists() && targetDir.list()?.any() == true) {
+                        targetDir.deleteRecursively()
                     }
 
-                    // Delete old resource container
-                    resourceMetadata.path.listFiles()
-                        ?.forEach { it.deleteRecursively() }
+                    directoryProvider.newFileReader(sourceFile).use { reader ->
+                        val entries = reader.list(".").toList()
+                        when {
+                            entries.size == 1 -> {
+                                // root is a directory, copy its content to avoid nested dirs
+                                reader.copyDirectory(entries.first(), targetDir)
+                            }
+
+                            else -> {
+                                reader.copyDirectory("/", targetDir)
+                            }
+                        }
+                    }
 
-                    val zip = resourceMetadata.path.resolve("$zipName.zip")
-                    tempZip.renameTo(zip)
+                    // Delete old resource container
+                    resourceMetadata.path.delete()
 
-                    val updatedRc = resourceMetadata.copy(path = zip)
+                    val updatedRc = resourceMetadata.copy(path = targetDir)
                     resourceMetadataRepo.update(updatedRc).blockingGet()
                 }
             }

common/src/main/kotlin/org/wycliffeassociates/otter/common/domain/project/exporter/resourcecontainer/BackupProjectExporter.kt

@@ -147,19 +147,31 @@ class BackupProjectExporter @Inject constructor(
      */
     private fun estimateSourceSize(workbook: Workbook): Long {
         val project = workbook.source.slug
-        val file = workbook.source.resourceMetadata.path
+        val sourceFile = workbook.source.resourceMetadata.path
         var size = 0L
 
-        ZipFile(file).use { zip ->
-            zip.entries()
-            .asIterator()
-            .forEach {
-                if (it.name.contains("${RcConstants.SOURCE_MEDIA_DIR}/${project}")) {
-                    size += it.compressedSize
-                }
+        if (sourceFile.isFile) {
+            ZipFile(sourceFile).use { zip ->
+                zip.entries()
+                    .asIterator()
+                    .forEach {
+                        if (it.name.contains("${RcConstants.SOURCE_MEDIA_DIR}/${project}")) {
+                            size += it.compressedSize
+                        }
+                    }
             }
+        } else {
+            sourceFile.walk()
+                .filter {
+                    it.invariantSeparatorsPath.contains("${RcConstants.SOURCE_MEDIA_DIR}/${project}") && it.isFile
+                }
+                .forEach {
+                    size += it.length()
+                }
+
         }
 
+
         return size
     }
 

common/src/main/kotlin/org/wycliffeassociates/otter/common/domain/project/importer/NewSourceImporter.kt

@@ -205,7 +205,7 @@ class NewSourceImporter @Inject constructor(
         return if (file.isDirectory) {
             copyRecursivelyToInternalDirectory(file, destinationDirectory)
         } else {
-            copyFileToInternalDirectory(file, destinationDirectory)
+            extractSourceToDir(file, destinationDirectory)
         }
     }
 
@@ -239,4 +239,21 @@ class NewSourceImporter @Inject constructor(
         }
         return destinationFile
     }
+
+    private fun extractSourceToDir(source: File, dir: File): File {
+        val targetDir = dir.resolve(source.nameWithoutExtension)
+        directoryProvider
+            .newFileReader(source)
+            .use { fileReader ->
+                fileReader.copyDirectory("/", targetDir)
+            }
+
+        targetDir.walk().forEach {
+            if (it.isDirectory && it.resolve("manifest.yaml").exists()) {
+                return it
+            }
+        }
+
+        return targetDir
+    }
 }

common/src/main/kotlin/org/wycliffeassociates/otter/common/domain/resourcecontainer/project/markdown/MarkdownProjectReader.kt

@@ -116,9 +116,15 @@ class MarkdownProjectReader(private val isHelp: Boolean) : IProjectReader {
             .toRelativeString(projectRoot)
             .substringBeforeLast('.')
             .split('/', '\\')
-            .asSequence()
-        val withSlug = sequenceOf(project.identifier) + fileParts.drop(1)
-        return withSlug.joinToString("_", transform = this::simplifyTitle)
+            .filter { it.isNotEmpty() } // ignore empty strings
+
+        val slugs = if (projectRoot is OtterFile.Z) {
+            listOf(project.identifier) + fileParts.drop(1)
+        } else {
+            listOf(project.identifier) + fileParts
+        }
+
+        return slugs.joinToString("_", transform = this::simplifyTitle)
     }
 
     private fun fileToSort(file: OtterFile) = when (file.nameWithoutExtension) {

jvm/workbookapp/src/integration-test/kotlin/integrationtest/initialization/TestInitializeSources.kt

@@ -80,10 +80,13 @@ class TestInitializeSources {
         testSub.assertComplete()
         testSub.assertNoErrors()
 
+        val sources = resourceMetadataRepository.getAllSources().blockingGet()
+
         Assert.assertEquals(init.version, database.installedEntityDao.fetchVersion(init))
         Assert.assertEquals(
-            1, resourceMetadataRepository.getAllSources().blockingGet().size
+            1, sources.size
         )
+        Assert.assertTrue(sources.all { it.path.isDirectory }) // sources are stored as directories
     }
 
     private fun prepareSource() {