From da250c764496717658354b04b9a452e2c10e937a Mon Sep 17 00:00:00 2001 From: Evgeny Date: Sun, 1 Oct 2023 11:51:38 +0300 Subject: [PATCH 01/10] refactor: add props for turning on/off dictionary api --- .../config/appprops/AppProperties.java | 3 ++- .../config/appprops/AppPropertiesConfig.java | 3 ++- .../config/appprops/ExternalApiParams.java | 16 ++++++++++++++++ .../config/appprops/ExternalApiProperties.java | 8 ++++++++ 4 files changed, 28 insertions(+), 2 deletions(-) create mode 100644 src/main/java/ru/dankoy/korvotoanki/config/appprops/ExternalApiParams.java create mode 100644 src/main/java/ru/dankoy/korvotoanki/config/appprops/ExternalApiProperties.java diff --git a/src/main/java/ru/dankoy/korvotoanki/config/appprops/AppProperties.java b/src/main/java/ru/dankoy/korvotoanki/config/appprops/AppProperties.java index 97d8749..29ba01d 100644 --- a/src/main/java/ru/dankoy/korvotoanki/config/appprops/AppProperties.java +++ b/src/main/java/ru/dankoy/korvotoanki/config/appprops/AppProperties.java @@ -11,7 +11,8 @@ @Getter @RequiredArgsConstructor @ConfigurationProperties(prefix = "korvo-to-anki") -public class AppProperties implements GoogleTranslatorProperties, DebugProperties, DictionaryApiProperties { +public class AppProperties implements GoogleTranslatorProperties, DebugProperties, + DictionaryApiProperties { private final String googleTranslatorUrl; diff --git a/src/main/java/ru/dankoy/korvotoanki/config/appprops/AppPropertiesConfig.java b/src/main/java/ru/dankoy/korvotoanki/config/appprops/AppPropertiesConfig.java index 9017236..49d72dc 100644 --- a/src/main/java/ru/dankoy/korvotoanki/config/appprops/AppPropertiesConfig.java +++ b/src/main/java/ru/dankoy/korvotoanki/config/appprops/AppPropertiesConfig.java @@ -5,7 +5,8 @@ import org.springframework.context.annotation.Configuration; @Configuration -@EnableConfigurationProperties({AppProperties.class, GoogleParamsProperties.class}) +@EnableConfigurationProperties({AppProperties.class, GoogleParamsProperties.class, + ExternalApiParams.class}) public class AppPropertiesConfig { } diff --git a/src/main/java/ru/dankoy/korvotoanki/config/appprops/ExternalApiParams.java b/src/main/java/ru/dankoy/korvotoanki/config/appprops/ExternalApiParams.java new file mode 100644 index 0000000..9f57040 --- /dev/null +++ b/src/main/java/ru/dankoy/korvotoanki/config/appprops/ExternalApiParams.java @@ -0,0 +1,16 @@ +package ru.dankoy.korvotoanki.config.appprops; + + +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import org.springframework.boot.context.properties.ConfigurationProperties; + + +@Getter +@RequiredArgsConstructor +@ConfigurationProperties(prefix = "korvo-to-anki.api") +public class ExternalApiParams implements ExternalApiProperties { + + private final boolean dictionaryApiEnabled; + +} diff --git a/src/main/java/ru/dankoy/korvotoanki/config/appprops/ExternalApiProperties.java b/src/main/java/ru/dankoy/korvotoanki/config/appprops/ExternalApiProperties.java new file mode 100644 index 0000000..bb277cf --- /dev/null +++ b/src/main/java/ru/dankoy/korvotoanki/config/appprops/ExternalApiProperties.java @@ -0,0 +1,8 @@ +package ru.dankoy.korvotoanki.config.appprops; + + +public interface ExternalApiProperties { + + boolean isDictionaryApiEnabled(); + +} From f4b7ebc2811f2dff9c2d831a6da5d206d13dbbc1 Mon Sep 17 00:00:00 2001 From: Evgeny Date: Sun, 1 Oct 2023 11:51:46 +0300 Subject: [PATCH 02/10] refactor: add enum for languages --- src/main/java/ru/dankoy/korvotoanki/config/Languages.java | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 src/main/java/ru/dankoy/korvotoanki/config/Languages.java diff --git a/src/main/java/ru/dankoy/korvotoanki/config/Languages.java b/src/main/java/ru/dankoy/korvotoanki/config/Languages.java new file mode 100644 index 0000000..4a03949 --- /dev/null +++ b/src/main/java/ru/dankoy/korvotoanki/config/Languages.java @@ -0,0 +1,5 @@ +package ru.dankoy.korvotoanki.config; + +public enum Languages { + EN, RU, AUTO +} From 6788cd40a615fb03376aad29537b820da41989da Mon Sep 17 00:00:00 2001 From: Evgeny Date: Sun, 1 Oct 2023 11:51:54 +0300 Subject: [PATCH 03/10] style: style --- .../ru/dankoy/korvotoanki/config/TemplateBuilderConfig.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/ru/dankoy/korvotoanki/config/TemplateBuilderConfig.java b/src/main/java/ru/dankoy/korvotoanki/config/TemplateBuilderConfig.java index 0fe0d96..4086082 100644 --- a/src/main/java/ru/dankoy/korvotoanki/config/TemplateBuilderConfig.java +++ b/src/main/java/ru/dankoy/korvotoanki/config/TemplateBuilderConfig.java @@ -11,7 +11,8 @@ public class TemplateBuilderConfig { @Bean - TemplateBuilder templateBuilder(@Value("${spring.freemarker.template-loader-path}") String templatesDir) { + TemplateBuilder templateBuilder( + @Value("${spring.freemarker.template-loader-path}") String templatesDir) { return new TemplateBuilderImpl(templatesDir); } From d2c8df6b477bcc7503d6063c682b1b1f60ae80e3 Mon Sep 17 00:00:00 2001 From: Evgeny Date: Sun, 1 Oct 2023 11:52:22 +0300 Subject: [PATCH 04/10] refactor: fixed npe --- .../korvotoanki/core/command/AnkiConverterCommand.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/main/java/ru/dankoy/korvotoanki/core/command/AnkiConverterCommand.java b/src/main/java/ru/dankoy/korvotoanki/core/command/AnkiConverterCommand.java index bfef880..5402660 100644 --- a/src/main/java/ru/dankoy/korvotoanki/core/command/AnkiConverterCommand.java +++ b/src/main/java/ru/dankoy/korvotoanki/core/command/AnkiConverterCommand.java @@ -5,6 +5,7 @@ import lombok.RequiredArgsConstructor; import org.springframework.shell.command.annotation.Command; import org.springframework.shell.command.annotation.Option; +import ru.dankoy.korvotoanki.core.domain.Title; import ru.dankoy.korvotoanki.core.domain.Vocabulary; import ru.dankoy.korvotoanki.core.service.converter.AnkiConverterService; import ru.dankoy.korvotoanki.core.service.objectmapper.ObjectMapperService; @@ -23,13 +24,13 @@ public class AnkiConverterCommand { description = "Translate and define text using google translator and dictionaryapi.dev") public String translateAndConvert( @Option(required = true, description = "text to translate") String text, - @Option(required = false, defaultValue = "auto", description = "source language") String sourceLanguage, + @Option(required = false, defaultValue = "en", description = "source language") String sourceLanguage, @Option(required = false, defaultValue = "ru", description = "target language") String targetLanguage, @Option(required = false, defaultValue = "t,at,md,rm", description = "options") String[] options ) { var vocabulary = new Vocabulary(text, - null, + new Title(0, null, 0L), 0L, 0L, 0L, @@ -39,7 +40,8 @@ public String translateAndConvert( 0L); List optionsList = Arrays.asList(options); - var ankiData = ankiConverterService.convert(vocabulary, sourceLanguage, targetLanguage, optionsList); + var ankiData = ankiConverterService.convert(vocabulary, sourceLanguage, targetLanguage, + optionsList); return objectMapperService.convertToString(ankiData); } From 78fb928a283345be8d8abd133e277af62b420896 Mon Sep 17 00:00:00 2001 From: Evgeny Date: Sun, 1 Oct 2023 11:52:33 +0300 Subject: [PATCH 05/10] refactor: default lang is en --- .../ru/dankoy/korvotoanki/core/command/AnkiExporterCommand.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/ru/dankoy/korvotoanki/core/command/AnkiExporterCommand.java b/src/main/java/ru/dankoy/korvotoanki/core/command/AnkiExporterCommand.java index e039bc8..a330c25 100644 --- a/src/main/java/ru/dankoy/korvotoanki/core/command/AnkiExporterCommand.java +++ b/src/main/java/ru/dankoy/korvotoanki/core/command/AnkiExporterCommand.java @@ -21,7 +21,7 @@ public class AnkiExporterCommand { alias = "ae", description = "Export to anki. Translate and define text using google translator and dictionaryapi.dev") public String export( - @Option(required = false, defaultValue = "auto", description = "source language") String sourceLanguage, + @Option(required = false, defaultValue = "en", description = "source language") String sourceLanguage, @Option(required = false, defaultValue = "ru", description = "target language") String targetLanguage, @Option(required = false, defaultValue = "t,at,md,rm", description = "options") String[] options ) { From 638e2270f7ba709a5203c819cfaac8097474c400 Mon Sep 17 00:00:00 2001 From: Evgeny Date: Sun, 1 Oct 2023 11:52:37 +0300 Subject: [PATCH 06/10] refactor: default lang is en --- .../dankoy/korvotoanki/core/command/GoogleTranslateCommand.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/ru/dankoy/korvotoanki/core/command/GoogleTranslateCommand.java b/src/main/java/ru/dankoy/korvotoanki/core/command/GoogleTranslateCommand.java index c4191ff..db8c1ff 100644 --- a/src/main/java/ru/dankoy/korvotoanki/core/command/GoogleTranslateCommand.java +++ b/src/main/java/ru/dankoy/korvotoanki/core/command/GoogleTranslateCommand.java @@ -22,7 +22,7 @@ public class GoogleTranslateCommand { description = "Translate text using google translate") public String translate( @Option(required = true, description = "text to translate") String text, - @Option(required = false, defaultValue = "auto", description = "source language") String sourceLanguage, + @Option(required = false, defaultValue = "en", description = "source language") String sourceLanguage, @Option(required = false, defaultValue = "ru", description = "target language") String targetLanguage, @Option(required = false, defaultValue = "t,at,md,rm", description = "options") String[] options ) { From 7b1e9c0e0f2f473ce34a59e4bc64cca12e159393 Mon Sep 17 00:00:00 2001 From: Evgeny Date: Sun, 1 Oct 2023 11:53:11 +0300 Subject: [PATCH 07/10] refactor: add ignore dictionaryapi --- .../converter/AnkiConverterServiceImpl.java | 31 ++++++++++++------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/src/main/java/ru/dankoy/korvotoanki/core/service/converter/AnkiConverterServiceImpl.java b/src/main/java/ru/dankoy/korvotoanki/core/service/converter/AnkiConverterServiceImpl.java index 76e3e98..45e7709 100644 --- a/src/main/java/ru/dankoy/korvotoanki/core/service/converter/AnkiConverterServiceImpl.java +++ b/src/main/java/ru/dankoy/korvotoanki/core/service/converter/AnkiConverterServiceImpl.java @@ -6,6 +6,8 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; +import ru.dankoy.korvotoanki.config.Languages; +import ru.dankoy.korvotoanki.config.appprops.ExternalApiProperties; import ru.dankoy.korvotoanki.core.domain.Vocabulary; import ru.dankoy.korvotoanki.core.domain.anki.AnkiData; import ru.dankoy.korvotoanki.core.domain.dictionaryapi.Word; @@ -24,26 +26,33 @@ public class AnkiConverterServiceImpl implements AnkiConverterService { private final DictionaryService dictionaryService; private final GoogleTranslator googleTranslator; private final AnkiDataFabric ankiDataFabric; + private final ExternalApiProperties externalApiProperties; public AnkiData convert(Vocabulary vocabulary, String sourceLanguage, String targetLanguage, List options) { List daResult = Collections.singletonList(Word.emptyWord()); - try { - daResult = dictionaryService.define(vocabulary.word()); - } catch (TooManyRequestsException e) { - log.warn("Hit rate limiter - {}. Going to sleep for 5 minutes and retry", e.getMessage()); - sleep(310000); - //todo: make advanced retry when too many requests + + var isDictionaryApiEnabled = externalApiProperties.isDictionaryApiEnabled(); + + // ignore auto source language because won't know the defined language. Look up for Tika Language Detection + if (isDictionaryApiEnabled && sourceLanguage.equals(Languages.EN.name().toLowerCase())) { + try { daResult = dictionaryService.define(vocabulary.word()); - } catch (DictionaryApiException e2) { - log.warn(String.format("Couldn't get definition from dictionaryapi.dev - %s", e2)); + } catch (TooManyRequestsException e) { + log.warn("Hit rate limiter - {}. Going to sleep for 5 minutes and retry", e.getMessage()); + sleep(310000); + //todo: make advanced retry when too many requests + try { + daResult = dictionaryService.define(vocabulary.word()); + } catch (DictionaryApiException e2) { + log.warn(String.format("Couldn't get definition from dictionaryapi.dev - %s", e2)); + } + } catch (DictionaryApiException e) { + log.warn(String.format("Couldn't get definition from dictionaryapi.dev - %s", e)); } } - catch (DictionaryApiException e) { - log.warn(String.format("Couldn't get definition from dictionaryapi.dev - %s", e)); - } var gtResult = googleTranslator.translate(vocabulary.word(), targetLanguage, sourceLanguage, options); From 4f1420d4d7a4cf653780bca49203c9f443980964 Mon Sep 17 00:00:00 2001 From: Evgeny Date: Sun, 1 Oct 2023 11:53:34 +0300 Subject: [PATCH 08/10] refactor: added default prop dictionaryApiEnabled --- src/main/resources/application.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index acf46ba..fba92f0 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -18,13 +18,15 @@ logging: korvo-to-anki: async: true + api: + dictionaryApiEnabled: true dictionary-api-url: "https://api.dictionaryapi.dev/api/v2/entries/en/" google-translator-url: "https://translate.googleapis.com/translate_a/single" trans-params: client: "gtx" # (using "t" raises 403 Forbidden) ie: "UTF-8" # input encoding oe: "UTF-8" # output encoding - sl: "auto" # source language (we need to specify "auto" to detect language) + sl: "en" # source language (we need to specify "auto" to detect language) tl: "ru" # target language hl: ${korvo-to-anki.trans-params.sl} # ? otf: 1 # ? From d227948d627701f038e64f0d670e8a8baba63aa9 Mon Sep 17 00:00:00 2001 From: Evgeny Date: Sun, 1 Oct 2023 11:54:58 +0300 Subject: [PATCH 09/10] refactor: updated readme --- README.md | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index e6f4d1f..ceebec3 100644 --- a/README.md +++ b/README.md @@ -31,11 +31,16 @@ database are going to be consumed in sync way (O(n)). In async mode all words in are going to be split in two lists and consumed in parallel in two threads. Default is async mode. -To change modes run jar with variable: **-Dkorvo-to-anki=sync** +To change modes run jar with variable: **-Dkorvo-to-anki.async=true** + +#### Available options of jar startup + + * korvo-to-anki.api.dictionaryApiEnabled: true/false - turn of or off dictionaryapi service + * korvo-to-anki.async: true/false - turn on or off async run #### On linux -`java -jar -Dkorvo-to-anki.async=true -Dspring.datasource.url=jdbc:sqlite:/path/to/vocabulary_builder.sqlite3 korvo-to-anki.jar ` +`java -jar Dkorvo-to-anki.async=false -Dkorvo-to-anki.api.dictionaryApiEnabled=false -Dspring.datasource.url=jdbc:sqlite:/path/to/vocabulary_builder.sqlite3 korvo-to-anki.jar ` #### On windows @@ -102,7 +107,6 @@ dictionaryapi. To show them in card one has to change card template. {{hint:Meaning}} ``` - #### Example result ```text @@ -128,6 +132,8 @@ minutes. When reached app fall asleep for 5 minutes and then retry to get word d word and run further as normal until requests limit is reached again. With all limitations export of 3080 words with sleep timeout was done in about _**30 minutes**_ +Dictionary api works only with english source language. Any other languages won't work same as auto. + ### Example ![img.png](screenshots/img.png) From 491b8a3522f394039945f45e6df8e1fde406406c Mon Sep 17 00:00:00 2001 From: Evgeny Date: Sun, 1 Oct 2023 12:03:02 +0300 Subject: [PATCH 10/10] style: style --- .../dankoy/korvotoanki/core/domain/anki/Meaning.java | 2 +- .../korvotoanki/core/fabric/anki/AnkiDataFabric.java | 3 ++- .../core/service/exporter/ExporterServiceAnki.java | 6 +----- .../core/service/googletrans/GoogleTranslator.java | 10 +++++----- src/main/resources/application.yml | 2 +- src/main/resources/templates/korvo-to-anki.ftl | 2 +- 6 files changed, 11 insertions(+), 14 deletions(-) diff --git a/src/main/java/ru/dankoy/korvotoanki/core/domain/anki/Meaning.java b/src/main/java/ru/dankoy/korvotoanki/core/domain/anki/Meaning.java index 5903b74..303ab56 100644 --- a/src/main/java/ru/dankoy/korvotoanki/core/domain/anki/Meaning.java +++ b/src/main/java/ru/dankoy/korvotoanki/core/domain/anki/Meaning.java @@ -3,7 +3,7 @@ import java.util.List; /** - * @param type noun, verb + * @param type noun, verb * @param definitions actual definitions * @param synonyms * @param antonyms diff --git a/src/main/java/ru/dankoy/korvotoanki/core/fabric/anki/AnkiDataFabric.java b/src/main/java/ru/dankoy/korvotoanki/core/fabric/anki/AnkiDataFabric.java index 7cd133e..192631e 100644 --- a/src/main/java/ru/dankoy/korvotoanki/core/fabric/anki/AnkiDataFabric.java +++ b/src/main/java/ru/dankoy/korvotoanki/core/fabric/anki/AnkiDataFabric.java @@ -8,6 +8,7 @@ public interface AnkiDataFabric { - AnkiData createAnkiData(Vocabulary vocabulary, GoogleTranslation googleTranslation, List word); + AnkiData createAnkiData(Vocabulary vocabulary, GoogleTranslation googleTranslation, + List word); } diff --git a/src/main/java/ru/dankoy/korvotoanki/core/service/exporter/ExporterServiceAnki.java b/src/main/java/ru/dankoy/korvotoanki/core/service/exporter/ExporterServiceAnki.java index 2a02fb8..3b72cf9 100644 --- a/src/main/java/ru/dankoy/korvotoanki/core/service/exporter/ExporterServiceAnki.java +++ b/src/main/java/ru/dankoy/korvotoanki/core/service/exporter/ExporterServiceAnki.java @@ -22,15 +22,11 @@ public class ExporterServiceAnki implements ExporterService { private static final int STEP_SIZE = 10; - private int counter = 0; - private final VocabularyService vocabularyService; - private final AnkiConverterService ankiConverterService; - private final TemplateCreatorService templateCreatorService; - private final IOService ioService; + private int counter = 0; @Override public void export(String sourceLanguage, String targetLanguage, List options) { diff --git a/src/main/java/ru/dankoy/korvotoanki/core/service/googletrans/GoogleTranslator.java b/src/main/java/ru/dankoy/korvotoanki/core/service/googletrans/GoogleTranslator.java index a9a9dc2..0e1c557 100644 --- a/src/main/java/ru/dankoy/korvotoanki/core/service/googletrans/GoogleTranslator.java +++ b/src/main/java/ru/dankoy/korvotoanki/core/service/googletrans/GoogleTranslator.java @@ -4,11 +4,11 @@ import ru.dankoy.korvotoanki.core.domain.googletranslation.GoogleTranslation; /* -* Used info from: -* - https://github.com/koreader/koreader/blob/34ba2fab301f43533ee6876d78809f685dd05614/frontend/ui/translator.lua#L4 -* - https://koreader.rocks/doc/modules/ui.translator.html -* - https://github.com/ssut/py-googletrans/blob/master/googletrans/client.py -* */ + * Used info from: + * - https://github.com/koreader/koreader/blob/34ba2fab301f43533ee6876d78809f685dd05614/frontend/ui/translator.lua#L4 + * - https://koreader.rocks/doc/modules/ui.translator.html + * - https://github.com/ssut/py-googletrans/blob/master/googletrans/client.py + * */ public interface GoogleTranslator { diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index fba92f0..d5c857b 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -32,7 +32,7 @@ korvo-to-anki: otf: 1 # ? ssel: 0 # ? tsel: 0 # ? - dt: # what we want in result + dt: # what we want in result - "t" # translation of source text - "at" # alternate translations - "md" # definitions of source text diff --git a/src/main/resources/templates/korvo-to-anki.ftl b/src/main/resources/templates/korvo-to-anki.ftl index 25aaab3..0f663ed 100644 --- a/src/main/resources/templates/korvo-to-anki.ftl +++ b/src/main/resources/templates/korvo-to-anki.ftl @@ -3,7 +3,7 @@ #deck column:1 #tags column:6 <#list ankiDataList as ankiData> -korvo-to-anki::${ankiData.book}|${ankiData.word}|${ankiData.translations?join(", ")}|${ankiData.myExample!""}|${ankiData.meanings}|${ankiData.book?replace(" ","_")} + korvo-to-anki::${ankiData.book}|${ankiData.word}|${ankiData.translations?join(", ")}|${ankiData.myExample!""}|${ankiData.meanings}|${ankiData.book?replace(" ","_")}