Skip to content

Commit

Permalink
fixed a crash when loading very large dictionaries on low memory devices
Browse files Browse the repository at this point in the history
  • Loading branch information
@sspanak
sspanak committed Feb 9, 2024
1 parent 5cdf979 commit ec87ce0
Show file tree
Hide file tree
Showing 3 changed files with 33 additions and 70 deletions.
88 changes: 21 additions & 67 deletions src/io/github/sspanak/tt9/db/DictionaryLoader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,8 @@ public class DictionaryLoader {
private ConsumerCompat<Bundle> onStatusChange;
private Thread loadThread;

private long importStartTime = 0;
private int currentFile = 0;
private long importStartTime = 0;
private long lastProgressUpdate = 0;


Expand Down Expand Up @@ -121,7 +121,6 @@ private void importAll(Language language) {
try {
long start = System.currentTimeMillis();
float progress = 1;
final float dictionaryMaxProgress = 90f;

sqlite.beginTransaction();

Expand All @@ -145,16 +144,10 @@ private void importAll(Language language) {
logLoadingStep("Custom words restored", language, start);

start = System.currentTimeMillis();
WordBatch words = readWordsFile(language, lettersCount, progress, progress + 25f);
progress += 25;
importWordFile(language, lettersCount, progress, 90);
progress = 90;
sendProgressMessage(language, progress, SettingsStore.DICTIONARY_IMPORT_PROGRESS_UPDATE_TIME);
logLoadingStep("Dictionary file loaded in memory", language, start);

start = System.currentTimeMillis();
saveWordBatch(words, progress, dictionaryMaxProgress, SettingsStore.DICTIONARY_IMPORT_PROGRESS_UPDATE_TIME);
progress = dictionaryMaxProgress;
sendProgressMessage(language, progress, 0);
logLoadingStep("Dictionary words saved in database", language, start);
logLoadingStep("Dictionary file imported", language, start);

start = System.currentTimeMillis();
Tables.createPositionIndex(sqlite.getDb(), language);
Expand Down Expand Up @@ -197,7 +190,7 @@ private void importAll(Language language) {
}


private int importLetters(Language language) throws InvalidLanguageCharactersException, DictionaryImportAbortedException {
private int importLetters(Language language) throws InvalidLanguageCharactersException {
int lettersCount = 0;
boolean isEnglish = language.getLocale().equals(Locale.ENGLISH);
WordBatch letters = new WordBatch(language);
Expand All @@ -210,13 +203,13 @@ private int importLetters(Language language) throws InvalidLanguageCharactersExc
}
}

saveWordBatch(letters, -1, -1, -1);
saveWordBatch(letters);

return lettersCount;
}


private WordBatch readWordsFile(Language language, int positionShift, float minProgress, float maxProgress) throws Exception {
private void importWordFile(Language language, int positionShift, float minProgress, float maxProgress) throws Exception {
int currentLine = 1;
int totalLines = getFileSize(language.getDictionaryFile());
float progressRatio = (maxProgress - minProgress) / totalLines;
Expand All @@ -235,74 +228,35 @@ private WordBatch readWordsFile(Language language, int positionShift, float minP
short frequency = getFrequency(parts);

try {
batch.add(word, frequency, currentLine + positionShift);
boolean isFinalized = batch.add(word, frequency, currentLine + positionShift);
if (isFinalized && batch.getWords().size() > SettingsStore.DICTIONARY_IMPORT_BATCH_SIZE) {
saveWordBatch(batch);
batch.clear();
}
} catch (InvalidLanguageCharactersException e) {
throw new DictionaryImportException(word, currentLine);
}

if (totalLines > 0 && currentLine % SettingsStore.DICTIONARY_IMPORT_PROGRESS_UPDATE_BATCH_SIZE == 0) {
if (totalLines > 0) {
sendProgressMessage(language, minProgress + progressRatio * currentLine, SettingsStore.DICTIONARY_IMPORT_PROGRESS_UPDATE_TIME);
}
}
}

return batch;
saveWordBatch(batch);
InsertOps.insertLanguageMeta(sqlite.getDb(), language.getId());
}


public void saveWordBatch(WordBatch batch, float minProgress, float maxProgress, int sizeUpdateInterval) throws DictionaryImportAbortedException {
float middleProgress = minProgress + (maxProgress - minProgress) / 2;

public void saveWordBatch(WordBatch batch) {
InsertOps insertOps = new InsertOps(sqlite.getDb(), batch.getLanguage());

insertWordsBatch(insertOps, batch, minProgress, middleProgress - 2, sizeUpdateInterval);
insertWordPositionsBatch(insertOps, batch, middleProgress - 2, maxProgress - 2, sizeUpdateInterval);
InsertOps.insertLanguageMeta(sqlite.getDb(), batch.getLanguage().getId());

if (sizeUpdateInterval > 0) {
sendProgressMessage(batch.getLanguage(), maxProgress, SettingsStore.DICTIONARY_IMPORT_PROGRESS_UPDATE_TIME);
for (int i = 0, end = batch.getWords().size(); i < end; i++) {
insertOps.insertWord(batch.getWords().get(i));
}
}


private void insertWordsBatch(InsertOps insertOps, WordBatch batch, float minProgress, float maxProgress, int sizeUpdateInterval) throws DictionaryImportAbortedException {
if (batch.getWords().size() == 0) {
return;
}

float progressRatio = (maxProgress - minProgress) / batch.getWords().size();

for (int progress = 0, end = batch.getWords().size(); progress < end; progress++) {
if (loadThread.isInterrupted()) {
sendProgressMessage(batch.getLanguage(), 0, 0);
throw new DictionaryImportAbortedException();
}

insertOps.insertWord(batch.getWords().get(progress));
if (sizeUpdateInterval > 0 && progress % sizeUpdateInterval == 0) {
sendProgressMessage(batch.getLanguage(), minProgress + progress * progressRatio, SettingsStore.DICTIONARY_IMPORT_PROGRESS_UPDATE_TIME);
}
}
}


private void insertWordPositionsBatch(InsertOps insertOps, WordBatch batch, float minProgress, float maxProgress, int sizeUpdateInterval) throws DictionaryImportAbortedException {
if (batch.getPositions().size() == 0) {
return;
}

float progressRatio = (maxProgress - minProgress) / batch.getPositions().size();

for (int progress = 0, end = batch.getPositions().size(); progress < end; progress++) {
if (loadThread.isInterrupted()) {
sendProgressMessage(batch.getLanguage(), 0, 0);
throw new DictionaryImportAbortedException();
}

insertOps.insertWordPosition(batch.getPositions().get(progress));
if (sizeUpdateInterval > 0 && progress % sizeUpdateInterval == 0) {
sendProgressMessage(batch.getLanguage(), minProgress + progress * progressRatio, SettingsStore.DICTIONARY_IMPORT_PROGRESS_UPDATE_TIME);
}
for (int i = 0, end = batch.getPositions().size(); i < end; i++) {
insertOps.insertWordPosition(batch.getPositions().get(i));
}
}

Expand Down Expand Up @@ -374,7 +328,7 @@ private void sendProgressMessage(Language language, float progress, int progress
Bundle progressMsg = new Bundle();
progressMsg.putInt("languageId", language.getId());
progressMsg.putLong("time", getImportTime());
progressMsg.putInt("progress", (int) Math.round(progress));
progressMsg.putInt("progress", Math.round(progress));
progressMsg.putInt("currentFile", currentFile);
asyncHandler.post(() -> onStatusChange.accept(progressMsg));
}
Expand Down
13 changes: 11 additions & 2 deletions src/io/github/sspanak/tt9/db/entities/WordBatch.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,11 +24,11 @@ public WordBatch(@NonNull Language language) {
this(language, 0);
}

public void add(@NonNull String word, int frequency, int position) throws InvalidLanguageCharactersException {
public boolean add(@NonNull String word, int frequency, int position) throws InvalidLanguageCharactersException {
words.add(Word.create(word, frequency, position));

if (position == 0) {
return;
return true;
}

String sequence = language.getDigitSequenceForWord(word);
Expand All @@ -44,7 +44,16 @@ public void add(@NonNull String word, int frequency, int position) throws Invali
positions.add(lastWordPosition);

lastWordPosition = WordPosition.create(sequence, position);

return true;
}

return false;
}

public void clear() {
words.clear();
positions.clear();
}

@NonNull public Language getLanguage() {
Expand Down
2 changes: 1 addition & 1 deletion src/io/github/sspanak/tt9/preferences/SettingsStore.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -272,7 +272,7 @@ public String getDoubleZeroChar() {

public boolean getDebugLogsEnabled() { return prefs.getBoolean("pref_enable_debug_logs", Logger.isDebugLevel()); }

public final static int DICTIONARY_IMPORT_PROGRESS_UPDATE_BATCH_SIZE = 1000; // items
public final static int DICTIONARY_IMPORT_BATCH_SIZE = 5000; // words
public final static int DICTIONARY_IMPORT_PROGRESS_UPDATE_TIME = 250; // ms
public final static int DICTIONARY_MISSING_WARNING_INTERVAL = 30000; // ms
public final static int PREFERENCES_CLICK_DEBOUNCE_TIME = 250; // ms
Expand Down

0 comments on commit ec87ce0

Please sign in to comment.