Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Dictionaryapi is optional. #31

Merged
merged 10 commits into from
Oct 1, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 9 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -31,11 +31,16 @@ database are going to be consumed in sync way (O(n)). In async mode all words in
are going to be split in two lists and consumed in parallel in two threads.
Default is async mode.

To change modes run jar with variable: **-Dkorvo-to-anki=sync**
To change modes run jar with variable: **-Dkorvo-to-anki.async=true**

#### Available options of jar startup

* korvo-to-anki.api.dictionaryApiEnabled: true/false - turn of or off dictionaryapi service
* korvo-to-anki.async: true/false - turn on or off async run

#### On linux

`java -jar -Dkorvo-to-anki.async=true -Dspring.datasource.url=jdbc:sqlite:/path/to/vocabulary_builder.sqlite3 korvo-to-anki.jar `
`java -jar Dkorvo-to-anki.async=false -Dkorvo-to-anki.api.dictionaryApiEnabled=false -Dspring.datasource.url=jdbc:sqlite:/path/to/vocabulary_builder.sqlite3 korvo-to-anki.jar `

#### On windows

Expand Down Expand Up @@ -102,7 +107,6 @@ dictionaryapi. To show them in card one has to change card template.
{{hint:Meaning}}
```


#### Example result

```text
Expand All @@ -128,6 +132,8 @@ minutes. When reached app fall asleep for 5 minutes and then retry to get word d
word and run further as normal until requests limit is reached again.
With all limitations export of 3080 words with sleep timeout was done in about _**30 minutes**_

Dictionary api works only with english source language. Any other languages won't work same as auto.

### Example

![img.png](screenshots/img.png)
Expand Down
5 changes: 5 additions & 0 deletions src/main/java/ru/dankoy/korvotoanki/config/Languages.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
package ru.dankoy.korvotoanki.config;

public enum Languages {
EN, RU, AUTO
}
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,8 @@ public class TemplateBuilderConfig {


@Bean
TemplateBuilder templateBuilder(@Value("${spring.freemarker.template-loader-path}") String templatesDir) {
TemplateBuilder templateBuilder(
@Value("${spring.freemarker.template-loader-path}") String templatesDir) {
return new TemplateBuilderImpl(templatesDir);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,8 @@
@Getter
@RequiredArgsConstructor
@ConfigurationProperties(prefix = "korvo-to-anki")
public class AppProperties implements GoogleTranslatorProperties, DebugProperties, DictionaryApiProperties {
public class AppProperties implements GoogleTranslatorProperties, DebugProperties,
DictionaryApiProperties {

private final String googleTranslatorUrl;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@
import org.springframework.context.annotation.Configuration;

@Configuration
@EnableConfigurationProperties({AppProperties.class, GoogleParamsProperties.class})
@EnableConfigurationProperties({AppProperties.class, GoogleParamsProperties.class,
ExternalApiParams.class})
public class AppPropertiesConfig {

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
package ru.dankoy.korvotoanki.config.appprops;


import lombok.Getter;
import lombok.RequiredArgsConstructor;
import org.springframework.boot.context.properties.ConfigurationProperties;


@Getter
@RequiredArgsConstructor
@ConfigurationProperties(prefix = "korvo-to-anki.api")
public class ExternalApiParams implements ExternalApiProperties {

private final boolean dictionaryApiEnabled;

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
package ru.dankoy.korvotoanki.config.appprops;


public interface ExternalApiProperties {

boolean isDictionaryApiEnabled();

}
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import lombok.RequiredArgsConstructor;
import org.springframework.shell.command.annotation.Command;
import org.springframework.shell.command.annotation.Option;
import ru.dankoy.korvotoanki.core.domain.Title;
import ru.dankoy.korvotoanki.core.domain.Vocabulary;
import ru.dankoy.korvotoanki.core.service.converter.AnkiConverterService;
import ru.dankoy.korvotoanki.core.service.objectmapper.ObjectMapperService;
Expand All @@ -23,13 +24,13 @@ public class AnkiConverterCommand {
description = "Translate and define text using google translator and dictionaryapi.dev")
public String translateAndConvert(
@Option(required = true, description = "text to translate") String text,
@Option(required = false, defaultValue = "auto", description = "source language") String sourceLanguage,
@Option(required = false, defaultValue = "en", description = "source language") String sourceLanguage,
@Option(required = false, defaultValue = "ru", description = "target language") String targetLanguage,
@Option(required = false, defaultValue = "t,at,md,rm", description = "options") String[] options
) {

var vocabulary = new Vocabulary(text,
null,
new Title(0, null, 0L),
0L,
0L,
0L,
Expand All @@ -39,7 +40,8 @@ public String translateAndConvert(
0L);

List<String> optionsList = Arrays.asList(options);
var ankiData = ankiConverterService.convert(vocabulary, sourceLanguage, targetLanguage, optionsList);
var ankiData = ankiConverterService.convert(vocabulary, sourceLanguage, targetLanguage,
optionsList);

return objectMapperService.convertToString(ankiData);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ public class AnkiExporterCommand {
alias = "ae",
description = "Export to anki. Translate and define text using google translator and dictionaryapi.dev")
public String export(
@Option(required = false, defaultValue = "auto", description = "source language") String sourceLanguage,
@Option(required = false, defaultValue = "en", description = "source language") String sourceLanguage,
@Option(required = false, defaultValue = "ru", description = "target language") String targetLanguage,
@Option(required = false, defaultValue = "t,at,md,rm", description = "options") String[] options
) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ public class GoogleTranslateCommand {
description = "Translate text using google translate")
public String translate(
@Option(required = true, description = "text to translate") String text,
@Option(required = false, defaultValue = "auto", description = "source language") String sourceLanguage,
@Option(required = false, defaultValue = "en", description = "source language") String sourceLanguage,
@Option(required = false, defaultValue = "ru", description = "target language") String targetLanguage,
@Option(required = false, defaultValue = "t,at,md,rm", description = "options") String[] options
) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
import java.util.List;

/**
* @param type noun, verb
* @param type noun, verb
* @param definitions actual definitions
* @param synonyms
* @param antonyms
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@

public interface AnkiDataFabric {

AnkiData createAnkiData(Vocabulary vocabulary, GoogleTranslation googleTranslation, List<Word> word);
AnkiData createAnkiData(Vocabulary vocabulary, GoogleTranslation googleTranslation,
List<Word> word);

}
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import ru.dankoy.korvotoanki.config.Languages;
import ru.dankoy.korvotoanki.config.appprops.ExternalApiProperties;
import ru.dankoy.korvotoanki.core.domain.Vocabulary;
import ru.dankoy.korvotoanki.core.domain.anki.AnkiData;
import ru.dankoy.korvotoanki.core.domain.dictionaryapi.Word;
Expand All @@ -24,26 +26,33 @@ public class AnkiConverterServiceImpl implements AnkiConverterService {
private final DictionaryService dictionaryService;
private final GoogleTranslator googleTranslator;
private final AnkiDataFabric ankiDataFabric;
private final ExternalApiProperties externalApiProperties;

public AnkiData convert(Vocabulary vocabulary, String sourceLanguage, String targetLanguage,
List<String> options) {

List<Word> daResult = Collections.singletonList(Word.emptyWord());
try {
daResult = dictionaryService.define(vocabulary.word());
} catch (TooManyRequestsException e) {
log.warn("Hit rate limiter - {}. Going to sleep for 5 minutes and retry", e.getMessage());
sleep(310000);
//todo: make advanced retry when too many requests

var isDictionaryApiEnabled = externalApiProperties.isDictionaryApiEnabled();

// ignore auto source language because won't know the defined language. Look up for Tika Language Detection
if (isDictionaryApiEnabled && sourceLanguage.equals(Languages.EN.name().toLowerCase())) {

try {
daResult = dictionaryService.define(vocabulary.word());
} catch (DictionaryApiException e2) {
log.warn(String.format("Couldn't get definition from dictionaryapi.dev - %s", e2));
} catch (TooManyRequestsException e) {
log.warn("Hit rate limiter - {}. Going to sleep for 5 minutes and retry", e.getMessage());
sleep(310000);
//todo: make advanced retry when too many requests
try {
daResult = dictionaryService.define(vocabulary.word());
} catch (DictionaryApiException e2) {
log.warn(String.format("Couldn't get definition from dictionaryapi.dev - %s", e2));
}
} catch (DictionaryApiException e) {
log.warn(String.format("Couldn't get definition from dictionaryapi.dev - %s", e));
}
}
catch (DictionaryApiException e) {
log.warn(String.format("Couldn't get definition from dictionaryapi.dev - %s", e));
}

var gtResult = googleTranslator.translate(vocabulary.word(), targetLanguage, sourceLanguage,
options);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,15 +22,11 @@
public class ExporterServiceAnki implements ExporterService {

private static final int STEP_SIZE = 10;
private int counter = 0;

private final VocabularyService vocabularyService;

private final AnkiConverterService ankiConverterService;

private final TemplateCreatorService templateCreatorService;

private final IOService ioService;
private int counter = 0;

@Override
public void export(String sourceLanguage, String targetLanguage, List<String> options) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,11 @@
import ru.dankoy.korvotoanki.core.domain.googletranslation.GoogleTranslation;

/*
* Used info from:
* - https://github.com/koreader/koreader/blob/34ba2fab301f43533ee6876d78809f685dd05614/frontend/ui/translator.lua#L4
* - https://koreader.rocks/doc/modules/ui.translator.html
* - https://github.com/ssut/py-googletrans/blob/master/googletrans/client.py
* */
* Used info from:
* - https://github.com/koreader/koreader/blob/34ba2fab301f43533ee6876d78809f685dd05614/frontend/ui/translator.lua#L4
* - https://koreader.rocks/doc/modules/ui.translator.html
* - https://github.com/ssut/py-googletrans/blob/master/googletrans/client.py
* */

public interface GoogleTranslator {

Expand Down
6 changes: 4 additions & 2 deletions src/main/resources/application.yml
Original file line number Diff line number Diff line change
Expand Up @@ -18,19 +18,21 @@ logging:

korvo-to-anki:
async: true
api:
dictionaryApiEnabled: true
dictionary-api-url: "https://api.dictionaryapi.dev/api/v2/entries/en/"
google-translator-url: "https://translate.googleapis.com/translate_a/single"
trans-params:
client: "gtx" # (using "t" raises 403 Forbidden)
ie: "UTF-8" # input encoding
oe: "UTF-8" # output encoding
sl: "auto" # source language (we need to specify "auto" to detect language)
sl: "en" # source language (we need to specify "auto" to detect language)
tl: "ru" # target language
hl: ${korvo-to-anki.trans-params.sl} # ?
otf: 1 # ?
ssel: 0 # ?
tsel: 0 # ?
dt: # what we want in result
dt: # what we want in result
- "t" # translation of source text
- "at" # alternate translations
- "md" # definitions of source text
Expand Down
2 changes: 1 addition & 1 deletion src/main/resources/templates/korvo-to-anki.ftl
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
#deck column:1
#tags column:6
<#list ankiDataList as ankiData>
korvo-to-anki::${ankiData.book}|${ankiData.word}|${ankiData.translations?join(", ")}|${ankiData.myExample!""}|${ankiData.meanings}|${ankiData.book?replace(" ","_")}
korvo-to-anki::${ankiData.book}|${ankiData.word}|${ankiData.translations?join(", ")}|${ankiData.myExample!""}|${ankiData.meanings}|${ankiData.book?replace(" ","_")}
</#list>