From 5e82a9be981ef455e24d906faec2c5755b72128c Mon Sep 17 00:00:00 2001 From: Joe Date: Mon, 1 Jul 2024 09:31:17 -0400 Subject: [PATCH] Import Audio Scripture Burrito (#1156) * wip * Update ScriptureBurritoUtils.kt * Fix Burrito Export * Import Burrito via conversion to RC * Update OratureFileFormat.kt * Update OratureFileFormat.kt * cleanup * wrap input streams in use blocks to close after transfer * More fixes * Update TestPersistenceComponent.kt * Update TestPersistenceComponent.kt --- .../common/data/ScriptureBurritoFileFormat.kt | 12 + .../domain/project/ImportProjectUseCase.kt | 5 + .../common/domain/project/ProjectFormat.kt | 1 + .../domain/project/ProjectFormatIdentifier.kt | 18 + .../project/importer/BurritoImporter.kt | 43 ++ .../importer/BurritoImporterFactory.kt | 24 + .../BurritoToResourceContainerConverter.kt | 417 ++++++++++++++++++ .../burrito/ScriptureBurritoUtils.kt | 70 +-- dependencies.gradle | 2 +- .../controls/dialog/ImportProjectDialog.kt | 6 +- .../ui/viewmodel/ImportProjectViewModel.kt | 17 +- 11 files changed, 584 insertions(+), 31 deletions(-) create mode 100644 common/src/main/kotlin/org/wycliffeassociates/otter/common/data/ScriptureBurritoFileFormat.kt create mode 100644 common/src/main/kotlin/org/wycliffeassociates/otter/common/domain/project/importer/BurritoImporter.kt create mode 100644 common/src/main/kotlin/org/wycliffeassociates/otter/common/domain/project/importer/BurritoImporterFactory.kt create mode 100644 common/src/main/kotlin/org/wycliffeassociates/otter/common/domain/resourcecontainer/burrito/BurritoToResourceContainerConverter.kt diff --git a/common/src/main/kotlin/org/wycliffeassociates/otter/common/data/ScriptureBurritoFileFormat.kt b/common/src/main/kotlin/org/wycliffeassociates/otter/common/data/ScriptureBurritoFileFormat.kt new file mode 100644 index 0000000000..269843b625 --- /dev/null +++ b/common/src/main/kotlin/org/wycliffeassociates/otter/common/data/ScriptureBurritoFileFormat.kt @@ -0,0 +1,12 @@ +package org.wycliffeassociates.otter.common.data + +enum class ScriptureBurritoFileFormat(val extension: String) { + BURRITO("burrito"), + ZIP("zip"); + + companion object { + val extensionList: List = entries.map { it.extension } + + fun isSupported(extension: String) = extension.lowercase() in extensionList + } +} \ No newline at end of file diff --git a/common/src/main/kotlin/org/wycliffeassociates/otter/common/domain/project/ImportProjectUseCase.kt b/common/src/main/kotlin/org/wycliffeassociates/otter/common/domain/project/ImportProjectUseCase.kt index fcdee20fba..3dcf9cd743 100644 --- a/common/src/main/kotlin/org/wycliffeassociates/otter/common/domain/project/ImportProjectUseCase.kt +++ b/common/src/main/kotlin/org/wycliffeassociates/otter/common/domain/project/ImportProjectUseCase.kt @@ -29,6 +29,7 @@ import io.reactivex.schedulers.Schedulers import org.slf4j.LoggerFactory import org.wycliffeassociates.otter.common.data.primitives.Language import org.wycliffeassociates.otter.common.data.primitives.ResourceMetadata +import org.wycliffeassociates.otter.common.domain.project.importer.BurritoImporterFactory import org.wycliffeassociates.otter.common.domain.project.importer.IProjectImporter import org.wycliffeassociates.otter.common.domain.project.importer.IProjectImporterFactory import org.wycliffeassociates.otter.common.domain.project.importer.ImportOptions @@ -49,6 +50,9 @@ const val SOURCE_PATH_TEMPLATE = "content/%s.zip" class ImportProjectUseCase @Inject constructor() { + @Inject + lateinit var burritoFactoryProvider: BurritoImporterFactory + @Inject lateinit var rcFactoryProvider: RCImporterFactory @@ -141,6 +145,7 @@ class ImportProjectUseCase @Inject constructor() { */ private fun getImporter(format: ProjectFormat): IProjectImporter { val factory: IProjectImporterFactory = when(format) { + ProjectFormat.SCRIPTURE_BURRITO -> burritoFactoryProvider ProjectFormat.RESOURCE_CONTAINER -> rcFactoryProvider ProjectFormat.TSTUDIO -> tsFactoryProvider else -> throw Exception("Unsupported project format.") diff --git a/common/src/main/kotlin/org/wycliffeassociates/otter/common/domain/project/ProjectFormat.kt b/common/src/main/kotlin/org/wycliffeassociates/otter/common/domain/project/ProjectFormat.kt index 5bf0c9879b..5437660a65 100644 --- a/common/src/main/kotlin/org/wycliffeassociates/otter/common/domain/project/ProjectFormat.kt +++ b/common/src/main/kotlin/org/wycliffeassociates/otter/common/domain/project/ProjectFormat.kt @@ -19,6 +19,7 @@ package org.wycliffeassociates.otter.common.domain.project enum class ProjectFormat { + SCRIPTURE_BURRITO, RESOURCE_CONTAINER, TSTUDIO, EPUB diff --git a/common/src/main/kotlin/org/wycliffeassociates/otter/common/domain/project/ProjectFormatIdentifier.kt b/common/src/main/kotlin/org/wycliffeassociates/otter/common/domain/project/ProjectFormatIdentifier.kt index 16d3b70732..e7fd216d3a 100644 --- a/common/src/main/kotlin/org/wycliffeassociates/otter/common/domain/project/ProjectFormatIdentifier.kt +++ b/common/src/main/kotlin/org/wycliffeassociates/otter/common/domain/project/ProjectFormatIdentifier.kt @@ -18,6 +18,8 @@ */ package org.wycliffeassociates.otter.common.domain.project +import org.bibletranslationtools.scriptureburrito.container.BurritoContainer +import org.wycliffeassociates.otter.common.domain.resourcecontainer.burrito.BurritoToResourceContainerConverter import org.wycliffeassociates.resourcecontainer.ResourceContainer import org.wycliffeassociates.tstudio2rc.Tstudio2RcConverter import java.io.File @@ -32,7 +34,10 @@ object ProjectFormatIdentifier { // set up the chains for identifying the project format val orature = OratureFileIdentifier() val tstudio = TstudioFileIdentifier() + val burrito = ScriptureBurritoFileIdentifier() + orature.next = tstudio + tstudio.next = burrito return orature } @@ -82,4 +87,17 @@ object ProjectFormatIdentifier { } } } + + private class ScriptureBurritoFileIdentifier : IFormatIdentifier { + override var next: IFormatIdentifier? = null + + override fun getFormat(file: File): ProjectFormat? { + return try { + BurritoContainer.load(file).close() + ProjectFormat.SCRIPTURE_BURRITO + } catch (e: Exception) { + next?.getFormat(file) + } + } + } } \ No newline at end of file diff --git a/common/src/main/kotlin/org/wycliffeassociates/otter/common/domain/project/importer/BurritoImporter.kt b/common/src/main/kotlin/org/wycliffeassociates/otter/common/domain/project/importer/BurritoImporter.kt new file mode 100644 index 0000000000..8694b73c5c --- /dev/null +++ b/common/src/main/kotlin/org/wycliffeassociates/otter/common/domain/project/importer/BurritoImporter.kt @@ -0,0 +1,43 @@ +package org.wycliffeassociates.otter.common.domain.project.importer + +import io.reactivex.Single +import io.reactivex.schedulers.Schedulers +import org.wycliffeassociates.otter.common.domain.resourcecontainer.ImportResult +import org.wycliffeassociates.otter.common.domain.resourcecontainer.burrito.BurritoToResourceContainerConverter +import org.wycliffeassociates.otter.common.persistence.IDirectoryProvider +import java.io.File +import javax.inject.Inject + +class BurritoImporter @Inject constructor( + private val directoryProvider: IDirectoryProvider, + private val converter: BurritoToResourceContainerConverter, +): IProjectImporter { + + private var next: RCImporter? = null + + override fun import( + burrito: File, + callback: ProjectImporterCallback?, + options: ImportOptions? + ): Single { + return Single + .fromCallable { + callback?.onNotifyProgress( + localizeKey = "converting_file", + percent = 10.0 + ) + val tempRc = directoryProvider.createTempFile("burrito_converted_rc", ".zip") + converter.convert(burrito, tempRc) + tempRc + } + .flatMap { fileToImport -> + next?.import(fileToImport, callback, options) + ?: Single.just(ImportResult.FAILED) + } + .subscribeOn(Schedulers.io()) + } + + fun setNext(next: RCImporter) { + this.next = next + } +} \ No newline at end of file diff --git a/common/src/main/kotlin/org/wycliffeassociates/otter/common/domain/project/importer/BurritoImporterFactory.kt b/common/src/main/kotlin/org/wycliffeassociates/otter/common/domain/project/importer/BurritoImporterFactory.kt new file mode 100644 index 0000000000..cb010d02b9 --- /dev/null +++ b/common/src/main/kotlin/org/wycliffeassociates/otter/common/domain/project/importer/BurritoImporterFactory.kt @@ -0,0 +1,24 @@ +package org.wycliffeassociates.otter.common.domain.project.importer + +import javax.inject.Inject +import javax.inject.Provider + +class BurritoImporterFactory @Inject constructor() : IProjectImporterFactory { + + @Inject lateinit var burritoImporter: Provider + @Inject lateinit var existingProjectImporter: Provider + @Inject lateinit var newSourceImporter: Provider + + private val importer: BurritoImporter by lazy { + val importer1 = burritoImporter.get() + val importer2 = existingProjectImporter.get() + val importer3 = newSourceImporter.get() + + importer1.setNext(importer2) + importer2.setNext(importer3) + + importer1 + } + + override fun makeImporter(): IProjectImporter = importer +} \ No newline at end of file diff --git a/common/src/main/kotlin/org/wycliffeassociates/otter/common/domain/resourcecontainer/burrito/BurritoToResourceContainerConverter.kt b/common/src/main/kotlin/org/wycliffeassociates/otter/common/domain/resourcecontainer/burrito/BurritoToResourceContainerConverter.kt new file mode 100644 index 0000000000..5ef7581bbf --- /dev/null +++ b/common/src/main/kotlin/org/wycliffeassociates/otter/common/domain/resourcecontainer/burrito/BurritoToResourceContainerConverter.kt @@ -0,0 +1,417 @@ +package org.wycliffeassociates.otter.common.domain.resourcecontainer.burrito + +import org.bibletranslationtools.scriptureburrito.IngredientSchema +import org.bibletranslationtools.scriptureburrito.MetadataSchema +import org.bibletranslationtools.scriptureburrito.container.BurritoContainer +import org.bibletranslationtools.scriptureburrito.container.accessors.IContainerAccessor +import org.bibletranslationtools.scriptureburrito.flavor.scripture.audio.AudioFlavorSchema +import org.bibletranslationtools.scriptureburrito.flavor.scripture.audio.AudioFormat +import org.bibletranslationtools.scriptureburrito.flavor.scripture.audio.Compression +import org.bibletranslationtools.scriptureburrito.flavor.scripture.audio.TrackConfiguration +import org.wycliffeassociates.resourcecontainer.IResourceContainerAccessor +import org.wycliffeassociates.resourcecontainer.ResourceContainer +import org.wycliffeassociates.resourcecontainer.entity.Checking +import org.wycliffeassociates.resourcecontainer.entity.DublinCore +import org.wycliffeassociates.resourcecontainer.entity.Language +import org.wycliffeassociates.resourcecontainer.entity.Manifest +import org.wycliffeassociates.resourcecontainer.entity.Media +import org.wycliffeassociates.resourcecontainer.entity.MediaManifest +import org.wycliffeassociates.resourcecontainer.entity.MediaProject +import org.wycliffeassociates.resourcecontainer.entity.Project +import java.io.File +import java.time.LocalDateTime +import java.time.ZoneId +import java.util.* +import java.util.zip.ZipFile +import java.util.zip.ZipOutputStream +import javax.inject.Inject +import kotlin.collections.HashMap + +internal typealias IngredientsByBook = Map>> + +internal val books = arrayOf( + "gen", "exo", "lev", "num", "deu", "jos", "jdg", "rut", "1sa", "2sa", "1ki", "2ki", "1ch", "2ch", + "ezr", "neh", "est", "job", "psa", "pro", "ecc", "sng", "isa", "jer", "lam", "ezk", "dan", "hos", + "jol", "amo", "oba", "jon", "mic", "nam", "hab", "zep", "hag", "zec", "mal", "mat", "mrk", "luk", + "jhn", "act", "rom", "1co", "2co", "gal", "eph", "php", "col", "1th", "2th", "1ti", "2ti", "tit", + "phm", "heb", "jas", "1pe", "2pe", "1jn", "2jn", "3jn", "jud", "rev" +) +internal val ot = books.slice(0 until 41) +internal val nt = books.slice(41 until 66) + +internal fun getBookSort(bookSlug: String): Int { + return books.indexOf(bookSlug) + 1 +} + +internal fun getTestament(bookSlug: String): String { + return when (bookSlug) { + in ot -> "bible-ot" + in nt -> "bible-nt" + else -> "" + } +} + +class BurritoToResourceContainerConverter @Inject constructor() { + + private val usfmFilenamePattern = "./{booknum}-{book}.usfm" + private val filenamePattern = "{language}_{title}_{book}_c{chapter}.{extension}" + private val DEFAULT_TITLE_CODE = "reg" + + fun convert( + burrito: File, + outputFile: File + ): Boolean { + if (outputFile.extension == "zip") outputFile.outputStream().use { ZipOutputStream(it).use { } } + val burrito = BurritoContainer.load(burrito) + burrito.use { + val metadata = it.manifest + ResourceContainer.create(outputFile) { + val (projects, media) = processContentInBurrito(metadata, burrito.accessor, this.accessor) + this.manifest = Manifest( + dublinCore = dublinCoreFromBurrito(metadata), + projects = projects, + checking = Checking(), + ) + this.media = media + this.write() + } + } + return true + } + + private fun dublinCoreFromBurrito(burrito: MetadataSchema): DublinCore { + val (identifier, title) = getTitleFromBurrito(burrito) + return DublinCore( + type = "bundle", + conformsTo = "0.2", + format = "text/usfm", + identifier = identifier, + title = title, + description = getDescriptionFromBurrito(burrito), + language = getLanguageFromBurrito(burrito), + rights = getCopyrightFromBurrito(burrito), + issued = getCreationDateFromBurrito(burrito), + modified = LocalDateTime.now().toString() + ) + } + + private fun getCreationDateFromBurrito(burrito: MetadataSchema): String { + return burrito + .meta + .dateCreated + .toInstant() + .atZone(ZoneId.systemDefault()) + .toLocalDateTime() + .toString() + } + + private fun getDescriptionFromBurrito(burrito: MetadataSchema): String { + + val langSlug = burrito.meta.defaultLocale + var desc = "" + burrito.identification?.let { + desc = it.description["en"] ?: it.description[langSlug] ?: "" + } + return desc + } + + private fun getTitleFromBurrito(burrito: MetadataSchema): Pair { + val langSlug = burrito.meta.defaultLocale + var slug = DEFAULT_TITLE_CODE + var title = "" + burrito.identification?.let { + slug = it.abbreviation["en"] ?: it.abbreviation[langSlug] ?: "" + title = it.name["en"] ?: it.name[langSlug] ?: "" + } + return Pair(slug, title) + } + + private fun getLanguageFromBurrito(burrito: MetadataSchema): Language { + val slug = burrito.meta.defaultLocale + val lang = burrito.languages.first { it.tag == slug } + val direction = lang.scriptDirection?.value() ?: "" + return Language( + direction, + slug, + lang.name[slug] ?: lang.name["en"] ?: "" + ) + } + + private fun getCopyrightFromBurrito(burrito: MetadataSchema): String { + return burrito + .copyright + .shortStatements + .map { it.statement } + .reduce { acc, shortStatement -> "$acc\n$shortStatement" } + } + + private fun processContentInBurrito( + burrito: MetadataSchema, + inputAccessor: IContainerAccessor, + outputAccessor: IResourceContainerAccessor + ): Pair, MediaManifest> { + val ingredientsByBook = getIngredientsByBook(burrito) + val usfmFilesByBook = getUSFMIngredients(ingredientsByBook) + val chapterAudioByBook = createChapterAudioIngredients(burrito, ingredientsByBook) + + val versification = getVersification(burrito, usfmFilesByBook, chapterAudioByBook) + + moveUSFMFiles(burrito, usfmFilesByBook, inputAccessor, outputAccessor) + moveAudioFiles(burrito, chapterAudioByBook, inputAccessor, outputAccessor) + + val mediaManifest = createMediaManifest(burrito, chapterAudioByBook) + val projects = createProjects( + burrito, + versification, + ingredientsByBook.keys, + usfmFilenamePattern + ) + + return Pair(projects, mediaManifest) + } + + private fun createChapterAudioIngredients( + burrito: MetadataSchema, + ingredientsByBook: IngredientsByBook + ): IngredientsByBook { + val filtered = filterAcceptedAudioFormats(burrito, ingredientsByBook) + val reconstructed = hashMapOf>>() + for ((book, ingredients) in filtered) { + val groupedByChapter = hashMapOf>>() + ingredients.forEach { item -> + val (file, ingredient) = item + val scope = ingredient.scope?.get(book.uppercase(Locale.US))!! + when { + scope.isEmpty() -> { + assert(false) + // breakBookAudioIntoChapters() + } + + scope.size == 1 -> { + val chapterNumber = scope.single().toInt() + if (groupedByChapter.containsKey(chapterNumber)) { + groupedByChapter[chapterNumber]!!.add(item) + } else { + groupedByChapter[chapterNumber] = mutableListOf(item) + } + } + + scope.size > 1 -> { + assert(false) + // combineSubchapterIntoChapter() + } + } + } + reconstructed[book] = groupedByChapter.values.flatten().toMutableList() + } + return reconstructed + } + + private fun filterAcceptedAudioFormats( + burrito: MetadataSchema, ingedientsByBook: IngredientsByBook + ): IngredientsByBook { + val audioFlavor = (burrito.type!!.flavorType.flavor as AudioFlavorSchema) + val approved = audioFlavor + .getFormats() + .filter { (formatName, format) -> + val supported = format.compression in arrayOf(Compression.WAV, Compression.MP3) + val validMp3 = validateMp3Format(format) + val validWav = validateWavFormat(format) + supported && (validMp3 || validWav) + } + val approvedMimeType = approved.map { (name, format) -> + when (format.compression) { + Compression.MP3 -> "audio/mpeg" + Compression.WAV -> "audio/wav" + else -> throw Exception("Audio format ${format} not filtered out.") + } + } + + val accepted = HashMap>>() + ingedientsByBook.forEach { (book, ingredients) -> + accepted[book] = ingredients.filter { (filename, ingredient) -> + ingredient.mimeType in listOf(*approvedMimeType.toTypedArray(), "application/x-cue") + } + } + return accepted + } + + private fun validateWavFormat(format: AudioFormat): Boolean { + return arrayOf( + format.compression == Compression.WAV, + // don't fail if sampling rate or configuration are not provided + format.samplingRate?.equals(44100) ?: true, + format.trackConfiguration?.equals(TrackConfiguration.MONO) ?: true, + format.bitDepth?.equals(16) ?: true + ).all { it } + } + + private fun validateMp3Format(format: AudioFormat): Boolean { + return arrayOf( + format.compression == Compression.MP3, + // don't fail if sampling rate or configuration are not provided + format.samplingRate?.equals(44100) ?: true, + format.trackConfiguration?.equals(TrackConfiguration.MONO) ?: true, + ).all { it } + } + + private fun createMediaManifest( + burrito: MetadataSchema, + chapterAudioByBook: IngredientsByBook + ): MediaManifest { + val (titleCode, _) = getTitleFromBurrito(burrito) + val languageCode = getLanguageFromBurrito(burrito).identifier + return MediaManifest( + projects = chapterAudioByBook.map { (book, chapterIngredients) -> + val audioEntries = chapterIngredients + .map { (chapterFile, _) -> + File(chapterFile).extension + } + .toSet() + .map { extension -> + Media( + identifier = extension, + chapterUrl = "media/${getFilename(languageCode, titleCode, book, extension)}" + ) + } + MediaProject( + identifier = book, + media = audioEntries + ) + } + ) + } + + private fun moveUSFMFiles( + burrito: MetadataSchema, + usfmFilesByBook: IngredientsByBook, + inputAccessor: IContainerAccessor, + outputAccessor: IResourceContainerAccessor + ) { + for ((book, usfmFiles) in usfmFilesByBook) { + if (usfmFiles.isEmpty()) continue + val bookIndex = books.indexOf(book.lowercase(Locale.US)) + // NT starts at 41 + val bookNumber = if (bookIndex <= 38) bookIndex + 1 else bookIndex + 2 + val (usfmFile, ingredient) = usfmFiles.first() + if (inputAccessor.fileExists(usfmFile)) { + val newPath = "$bookNumber-${book.uppercase(Locale.US)}.usfm" + inputAccessor.getInputStream(usfmFile).use { ifs -> + outputAccessor.write(newPath) { + ifs.transferTo(it) + } + } + } + } + } + + private fun moveAudioFiles( + burrito: MetadataSchema, + usfmFilesByBook: IngredientsByBook, + inputAccessor: IContainerAccessor, + outputAccessor: IResourceContainerAccessor + ) { + val (titleCode, _) = getTitleFromBurrito(burrito) + val languageCode = getLanguageFromBurrito(burrito).identifier + for ((book, audioFiles) in usfmFilesByBook) { + if (audioFiles.isEmpty()) continue + val bookIndex = books.indexOf(book.lowercase(Locale.US)) + // NT starts at 41 + val bookNumber = if (bookIndex <= 38) bookIndex + 1 else bookIndex + 2 + for (af in audioFiles) { + val (audioFile, ingredient) = af + val chapter = ingredient!!.scope?.get(book.uppercase(Locale.US))?.single()!! + val extension = File(audioFile).extension + if (inputAccessor.fileExists(audioFile)) { + val newPath = "media/${ + getFilename(languageCode, titleCode, book, extension) + .replace("{chapter}", chapter) + }" + inputAccessor.getInputStream(audioFile).use { ifs -> + outputAccessor.write(newPath) { + ifs.transferTo(it) + } + } + } + } + } + } + + private fun getVersification( + burrito: MetadataSchema, + usfmFilesByBook: Any, + chapterAudioByBook: Any + ): String { + return "ufw" + } + + private fun getUSFMIngredients(ingedientsByBook: IngredientsByBook): IngredientsByBook { + val usfmMimetypes = listOf("text/usfm", "text/usfm3") + val filtered = HashMap>>() + ingedientsByBook.forEach { book, ingredientList -> + val items = ingredientList.filter { (file, ingredient) -> + File(file).extension == "usfm" || ingredient.mimeType in usfmMimetypes + } + filtered[book] = items + } + return filtered + } + + private fun createProjects( + burrito: MetadataSchema, + versification: String, + bookSlugs: Iterable, + filenamePattern: String + ): List { + return bookSlugs.map { slug -> + val usfmFile = filenamePattern + .replace("{booknum}", "${getBookSort(slug)}") + .replace("{book}", slug.uppercase(Locale.US)) + Project( + title = getBookTitle(burrito, slug), + versification = versification, + identifier = slug, + sort = getBookSort(slug), + path = usfmFile, + categories = listOf(getTestament(slug)) + ) + } + } + + private fun getBookTitle(burrito: MetadataSchema, bookSlug: String): String { + val locale = burrito.meta.defaultLocale + val localizedTitle = burrito.localizedNames["book-${bookSlug.lowercase(Locale.US)}"] + localizedTitle?.let { localizedTitle -> + return localizedTitle.short[locale] ?: localizedTitle.short["en"] ?: "" + } + return "" + } + + private fun getFilename( + languageCode: String, + titleCode: String, + bookSlug: String, + extension: String + ): String { + val titleCode = if (titleCode.isEmpty()) DEFAULT_TITLE_CODE else titleCode + return filenamePattern + .replace("{book}", bookSlug) + .replace("{title}", titleCode) + .replace("{language}", languageCode) + .replace("{extension}", extension) + } + + private fun getIngredientsByBook(burrito: MetadataSchema): IngredientsByBook { + val slugs = burrito.type!!.flavorType.currentScope.keys.map { it.lowercase(Locale.US) } + val ingredientsByBook = slugs.associateWith { mutableListOf>() } + burrito.ingredients.forEach { filepath, item -> + item.scope?.let { scope -> + scope.keys.forEach { + val slug = it.lowercase(Locale.US) + ingredientsByBook[slug]?.add(Pair(filepath, item)) + } + } + } + return ingredientsByBook + } +} \ No newline at end of file diff --git a/common/src/main/kotlin/org/wycliffeassociates/otter/common/domain/resourcecontainer/burrito/ScriptureBurritoUtils.kt b/common/src/main/kotlin/org/wycliffeassociates/otter/common/domain/resourcecontainer/burrito/ScriptureBurritoUtils.kt index 811cf89022..f4d9b897b5 100644 --- a/common/src/main/kotlin/org/wycliffeassociates/otter/common/domain/resourcecontainer/burrito/ScriptureBurritoUtils.kt +++ b/common/src/main/kotlin/org/wycliffeassociates/otter/common/domain/resourcecontainer/burrito/ScriptureBurritoUtils.kt @@ -5,11 +5,8 @@ import com.fasterxml.jackson.databind.SerializationFeature import com.fasterxml.jackson.module.kotlin.registerKotlinModule import org.bibletranslationtools.scriptureburrito.Checksum import org.bibletranslationtools.scriptureburrito.CopyrightSchema -import org.bibletranslationtools.scriptureburrito.DerivedMetaSchema -import org.bibletranslationtools.scriptureburrito.DerivedMetadataSchema import org.bibletranslationtools.scriptureburrito.Flavor import org.bibletranslationtools.scriptureburrito.Format -import org.bibletranslationtools.scriptureburrito.IdentificationSchema import org.bibletranslationtools.scriptureburrito.IngredientSchema import org.bibletranslationtools.scriptureburrito.IngredientsSchema import org.bibletranslationtools.scriptureburrito.LanguageSchema @@ -18,7 +15,6 @@ import org.bibletranslationtools.scriptureburrito.LocalizedNamesSchema import org.bibletranslationtools.scriptureburrito.LocalizedText import org.bibletranslationtools.scriptureburrito.MetaVersionSchema import org.bibletranslationtools.scriptureburrito.MetadataSchema -import org.bibletranslationtools.scriptureburrito.PrimaryIdentification import org.bibletranslationtools.scriptureburrito.ScopeSchema import org.bibletranslationtools.scriptureburrito.ShortStatement import org.bibletranslationtools.scriptureburrito.SoftwareAndUserInfoSchema @@ -35,7 +31,6 @@ import org.wycliffeassociates.otter.common.data.IAppInfo import org.wycliffeassociates.otter.common.data.workbook.Workbook import org.wycliffeassociates.otter.common.domain.resourcecontainer.RcConstants import org.wycliffeassociates.otter.common.domain.resourcecontainer.burrito.auth.AuthProvider -import org.wycliffeassociates.otter.common.domain.resourcecontainer.burrito.auth.IdAuthorityProvider import org.wycliffeassociates.otter.common.persistence.IDirectoryProvider import org.wycliffeassociates.resourcecontainer.ResourceContainer import java.io.File @@ -104,26 +99,30 @@ class ScriptureBurritoUtils @Inject constructor( } ), idAuthorityProvider.createIdAuthority(), - idAuthorityProvider.createIdentification(), + idAuthorityProvider.createIdentification().apply { + this.name["en"] = workbook.target.resourceMetadata.title + this.abbreviation["en"] = workbook.target.resourceMetadata.identifier + }, confidential = false, copyright = CopyrightSchema().apply { - this.shortStatements = mutableListOf(ShortStatement(rc.manifest.dublinCore.rights, langCode)) + this.shortStatements = + mutableListOf(ShortStatement(rc.manifest.dublinCore.rights, langCode)) }, type = TypeSchema( - FlavorType(Flavor.SCRIPTURE).apply { - val formats = Formats() - formats.put("format-wav", AudioFormat(Compression.WAV)) - formats.put("format-mp3", AudioFormat(Compression.MP3)) - this.flavor = AudioFlavorSchema( + FlavorType( + name = Flavor.SCRIPTURE, + AudioFlavorSchema( mutableSetOf(Performance.READING, Performance.SINGLE_VOICE), - formats - ).apply { - name = "audioTranslation" - currentScope = ScopeSchema().apply { - this[workbook.target.slug.uppercase(Locale.US)] = takes.keys.map { "$it" }.toMutableList() + formats = Formats().apply { + put("format-wav", AudioFormat(Compression.WAV)) + put("format-mp3", AudioFormat(Compression.MP3)) } + ), + currentScope = ScopeSchema().apply { + this[workbook.target.slug.uppercase(Locale.US)] = + takes.keys.map { "$it" }.toMutableList() } - } + ) ), languages = Languages().apply { add( @@ -149,19 +148,33 @@ class ScriptureBurritoUtils @Inject constructor( val ingredients = IngredientsSchema() val files = mutableMapOf() val outTempDir = File(tempDir, "burritoDir").apply { mkdirs() } - val usfmFiles = rc.manifest.projects.map { - if (it.path.contains(".usfm")) { - val path = "${it.path.removePrefix("./")}" - val bookDir = File(outTempDir, "${it.identifier}").mkdirs() + val usfmFiles = rc.manifest.projects.map { project -> + if (project.path.contains(".usfm")) { + val path = "${project.path.removePrefix("./")}" + val bookDir = File(outTempDir, "${project.identifier}").mkdirs() val outFile = File(outTempDir, path).apply { createNewFile() } - rc.accessor.getInputStream(it.path.removePrefix("./")).transferTo(outFile.outputStream()) - files["${it.identifier}/$path"] = outFile + + rc.accessor + .getInputStream(project.path.removePrefix("./")).use { ifs -> + outFile.outputStream().use { ofs -> + ifs.transferTo(ofs) + } + } + + files["${project.identifier}/$path"] = outFile val ingredient = IngredientSchema().apply { this.mimeType = "text/usfm" this.size = outFile.length().toInt() this.checksum = Checksum().apply { this.md5 = calculateMD5(outFile) } + this.scope = ScopeSchema().apply { + put( + project.identifier.uppercase(Locale.US), + mutableListOf() + ) + } + } ingredients["$path"] = ingredient } @@ -169,7 +182,7 @@ class ScriptureBurritoUtils @Inject constructor( val book = workbook.target.slug takes.forEach { (chapterNumber, audioFile) -> for (take in audioFile) { - val path = "${RcConstants.MEDIA_DIR}/${take.name}" + val path = "${RcConstants.SOURCE_MEDIA_DIR}/${take.name}" val outFile = File(outTempDir, path).apply { parentFile.mkdirs() } files[path] = outFile val ingredient = IngredientSchema().apply { @@ -183,7 +196,12 @@ class ScriptureBurritoUtils @Inject constructor( this.checksum = Checksum().apply { this.md5 = calculateMD5(take) } - scope = ScopeSchema().apply { put(book.uppercase(Locale.US), mutableListOf("$chapterNumber")) } + scope = ScopeSchema().apply { + put( + book.uppercase(Locale.US), + mutableListOf("$chapterNumber") + ) + } } ingredients[path] = ingredient } diff --git a/dependencies.gradle b/dependencies.gradle index 1f2eb56bf1..afd5999fb7 100644 --- a/dependencies.gradle +++ b/dependencies.gradle @@ -39,7 +39,7 @@ ext { commonmarkVer = '0.12.1' clapperJavaUtilVer = '3.2.0' kotlinresourcecontainerVer = '0.12.0' - kotlinscriptureburritoVer = "1.0.0" + kotlinscriptureburritoVer = "1.0.1" slf4jApiVer = '2.0.13' log4j2Ver = '2.15.0' ikonliVer = '12.2.0' diff --git a/jvm/controls/src/main/kotlin/org/wycliffeassociates/otter/jvm/controls/dialog/ImportProjectDialog.kt b/jvm/controls/src/main/kotlin/org/wycliffeassociates/otter/jvm/controls/dialog/ImportProjectDialog.kt index a72e4c35be..25751d2551 100644 --- a/jvm/controls/src/main/kotlin/org/wycliffeassociates/otter/jvm/controls/dialog/ImportProjectDialog.kt +++ b/jvm/controls/src/main/kotlin/org/wycliffeassociates/otter/jvm/controls/dialog/ImportProjectDialog.kt @@ -29,6 +29,8 @@ import org.kordamp.ikonli.javafx.FontIcon import org.kordamp.ikonli.materialdesign.MaterialDesign import org.slf4j.LoggerFactory import org.wycliffeassociates.otter.common.data.OratureFileFormat +import org.wycliffeassociates.otter.common.data.ScriptureBurritoFileFormat +import org.wycliffeassociates.otter.common.data.TstudioFileFormat import org.wycliffeassociates.otter.jvm.controls.event.ProjectImportEvent import tornadofx.* import java.io.File @@ -106,7 +108,9 @@ class ImportProjectDialog : OtterDialog() { arrayOf( FileChooser.ExtensionFilter( messages["oratureFileTypes"], - *OratureFileFormat.extensionList.map { "*.$it" }.toTypedArray() + *OratureFileFormat.extensionList.map { "*.$it" }.toTypedArray(), + *ScriptureBurritoFileFormat.extensionList.map { "*.$it" }.toTypedArray(), + *TstudioFileFormat.extensionList.map { "*.$it" }.toTypedArray(), ) ), mode = FileChooserMode.Single, diff --git a/jvm/workbookapp/src/main/kotlin/org/wycliffeassociates/otter/jvm/workbookapp/ui/viewmodel/ImportProjectViewModel.kt b/jvm/workbookapp/src/main/kotlin/org/wycliffeassociates/otter/jvm/workbookapp/ui/viewmodel/ImportProjectViewModel.kt index 4950913c47..c269997296 100644 --- a/jvm/workbookapp/src/main/kotlin/org/wycliffeassociates/otter/jvm/workbookapp/ui/viewmodel/ImportProjectViewModel.kt +++ b/jvm/workbookapp/src/main/kotlin/org/wycliffeassociates/otter/jvm/workbookapp/ui/viewmodel/ImportProjectViewModel.kt @@ -31,6 +31,7 @@ import org.slf4j.LoggerFactory import org.wycliffeassociates.otter.jvm.workbookapp.ui.model.ConflictResolution import org.wycliffeassociates.otter.common.data.OratureFileFormat import org.wycliffeassociates.otter.common.data.ProgressStatus +import org.wycliffeassociates.otter.common.data.ScriptureBurritoFileFormat import org.wycliffeassociates.otter.common.data.TstudioFileFormat import org.wycliffeassociates.otter.common.data.workbook.WorkbookDescriptor import org.wycliffeassociates.otter.common.domain.project.ImportProjectUseCase @@ -57,8 +58,11 @@ class ImportProjectViewModel : ViewModel() { val settingsViewModel: SettingsViewModel by inject() - @Inject lateinit var directoryProvider: IDirectoryProvider - @Inject lateinit var importProjectProvider : Provider + @Inject + lateinit var directoryProvider: IDirectoryProvider + + @Inject + lateinit var importProjectProvider: Provider val showImportSuccessDialogProperty = SimpleBooleanProperty(false) val showImportErrorDialogProperty = SimpleBooleanProperty(false) @@ -174,6 +178,7 @@ class ImportProjectViewModel : ViewModel() { ) false } + files.first().isDirectory -> { snackBarObservable.onNext(messages["importDirectoryError"]) logger.error( @@ -181,13 +186,19 @@ class ImportProjectViewModel : ViewModel() { ) false } - files.first().extension !in (OratureFileFormat.extensionList + TstudioFileFormat.extensionList) -> { + + files.first().extension !in ( + OratureFileFormat.extensionList + + TstudioFileFormat.extensionList + + ScriptureBurritoFileFormat.extensionList + ) -> { snackBarObservable.onNext(messages["importInvalidFileError"]) logger.error( "(Drag-Drop) Invalid import file extension. Input files: ${files.first()}" ) false } + else -> true } }