diff --git a/java/src/com/android/inputmethod/latin/BinaryDictionary.java b/java/src/com/android/inputmethod/latin/BinaryDictionary.java
index 632ee0da4b3962298c8d9fdc369023fab229e5ee..61ccfcfad31ba2a5fcbaefa6287df16ec25e873a 100644
--- a/java/src/com/android/inputmethod/latin/BinaryDictionary.java
+++ b/java/src/com/android/inputmethod/latin/BinaryDictionary.java
@@ -27,6 +27,7 @@ import com.android.inputmethod.latin.utils.CollectionUtils;
import com.android.inputmethod.latin.utils.JniUtils;
import com.android.inputmethod.latin.utils.StringUtils;
+import java.io.File;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Locale;
@@ -244,11 +245,18 @@ public final class BinaryDictionary extends Dictionary {
return getBigramProbabilityNative(mNativeDict, codePoints0, codePoints1);
}
+ private void runGCIfRequired() {
+ if (needsToRunGCNative(mNativeDict)) {
+ flushWithGC();
+ }
+ }
+
// Add a unigram entry to binary dictionary in native code.
public void addUnigramWord(final String word, final int probability) {
if (TextUtils.isEmpty(word)) {
return;
}
+ runGCIfRequired();
final int[] codePoints = StringUtils.toCodePointArray(word);
addUnigramWordNative(mNativeDict, codePoints, probability);
}
@@ -258,6 +266,7 @@ public final class BinaryDictionary extends Dictionary {
if (TextUtils.isEmpty(word0) || TextUtils.isEmpty(word1)) {
return;
}
+ runGCIfRequired();
final int[] codePoints0 = StringUtils.toCodePointArray(word0);
final int[] codePoints1 = StringUtils.toCodePointArray(word1);
addBigramWordsNative(mNativeDict, codePoints0, codePoints1, probability);
@@ -268,24 +277,30 @@ public final class BinaryDictionary extends Dictionary {
if (TextUtils.isEmpty(word0) || TextUtils.isEmpty(word1)) {
return;
}
+ runGCIfRequired();
final int[] codePoints0 = StringUtils.toCodePointArray(word0);
final int[] codePoints1 = StringUtils.toCodePointArray(word1);
removeBigramWordsNative(mNativeDict, codePoints0, codePoints1);
}
- @UsedForTesting
public void flush() {
if (!isValidDictionary()) return;
flushNative(mNativeDict, mDictFilePath);
+ closeNative(mNativeDict);
+ final File dictFile = new File(mDictFilePath);
+ mNativeDict = openNative(dictFile.getAbsolutePath(), 0 /* startOffset */,
+ dictFile.length(), true /* isUpdatable */);
}
- @UsedForTesting
public void flushWithGC() {
if (!isValidDictionary()) return;
flushWithGCNative(mNativeDict, mDictFilePath);
+ closeNative(mNativeDict);
+ final File dictFile = new File(mDictFilePath);
+ mNativeDict = openNative(dictFile.getAbsolutePath(), 0 /* startOffset */,
+ dictFile.length(), true /* isUpdatable */);
}
- @UsedForTesting
public boolean needsToRunGC() {
if (!isValidDictionary()) return false;
return needsToRunGCNative(mNativeDict);
diff --git a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java
index fcd7ede1a9198e4087e662aa2e006a07654a6209..0774ce20311b27e178167ff070465f8b166842db 100644
--- a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java
+++ b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java
@@ -22,14 +22,22 @@ import android.util.Log;
import com.android.inputmethod.annotations.UsedForTesting;
import com.android.inputmethod.keyboard.ProximityInfo;
-import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo;
+import com.android.inputmethod.latin.makedict.DictEncoder;
+import com.android.inputmethod.latin.makedict.FormatSpec;
+import com.android.inputmethod.latin.makedict.FusionDictionary;
+import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray;
+import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
+import com.android.inputmethod.latin.makedict.Ver3DictEncoder;
import com.android.inputmethod.latin.personalization.DynamicPersonalizationDictionaryWriter;
+import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo;
import com.android.inputmethod.latin.utils.AsyncResultHolder;
import com.android.inputmethod.latin.utils.CollectionUtils;
import com.android.inputmethod.latin.utils.PrioritizedSerialExecutor;
import java.io.File;
+import java.io.IOException;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicReference;
@@ -49,9 +57,9 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
/** Whether to print debug output to log */
private static boolean DEBUG = false;
- // TODO: Remove and enable dynamic update in native code.
+ // TODO: Remove.
/** Whether to call binary dictionary dynamically updating methods. */
- private static boolean ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE = false;
+ public static boolean ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE = true;
private static final int TIMEOUT_FOR_READ_OPS_IN_MILLISECONDS = 100;
@@ -60,6 +68,9 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
*/
protected static final int MAX_WORD_LENGTH = Constants.DICTIONARY_MAX_WORD_LENGTH;
+ private static final FormatSpec.FormatOptions FORMAT_OPTIONS =
+ new FormatSpec.FormatOptions(3 /* version */, true /* supportsDynamicUpdate */);
+
/**
* A static map of time recorders, each of which records the time of accesses to a single binary
* dictionary file. The key for this map is the filename and the value is the shared dictionary
@@ -154,7 +165,11 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
private static AbstractDictionaryWriter getDictionaryWriter(final Context context,
final String dictType, final boolean isDynamicPersonalizationDictionary) {
if (isDynamicPersonalizationDictionary) {
- return new DynamicPersonalizationDictionaryWriter(context, dictType);
+ if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) {
+ return null;
+ } else {
+ return new DynamicPersonalizationDictionaryWriter(context, dictType);
+ }
} else {
return new DictionaryWriter(context, dictType);
}
@@ -198,7 +213,9 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
mBinaryDictionary.close();
mBinaryDictionary = null;
}
- mDictionaryWriter.close();
+ if (mDictionaryWriter != null) {
+ mDictionaryWriter.close();
+ }
}
});
}
@@ -220,7 +237,23 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
getExecutor(mFilename).execute(new Runnable() {
@Override
public void run() {
- mDictionaryWriter.clear();
+ if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE && mDictionaryWriter == null) {
+ mBinaryDictionary.close();
+ final File file = new File(mContext.getFilesDir(), mFilename);
+ final FusionDictionary dict = new FusionDictionary(new PtNodeArray(),
+ new FusionDictionary.DictionaryOptions(new HashMap<String,String>(),
+ false, false));
+ final DictEncoder dictEncoder = new Ver3DictEncoder(file);
+ try {
+ dictEncoder.writeDictionary(dict, FORMAT_OPTIONS);
+ } catch (IOException e) {
+ Log.e(TAG, "Exception in creating new dictionary file.", e);
+ } catch (UnsupportedFormatException e) {
+ Log.e(TAG, "Exception in creating new dictionary file.", e);
+ }
+ } else {
+ mDictionaryWriter.clear();
+ }
}
});
}
@@ -257,9 +290,10 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
public void run() {
if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) {
mBinaryDictionary.addUnigramWord(word, frequency);
+ } else {
+ // TODO: Remove.
+ mDictionaryWriter.addUnigramWord(word, shortcutTarget, frequency, isNotAWord);
}
- // TODO: Remove.
- mDictionaryWriter.addUnigramWord(word, shortcutTarget, frequency, isNotAWord);
}
});
}
@@ -280,10 +314,11 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
public void run() {
if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) {
mBinaryDictionary.addBigramWords(word0, word1, frequency);
+ } else {
+ // TODO: Remove.
+ mDictionaryWriter.addBigramWords(word0, word1, frequency, isValid,
+ 0 /* lastTouchedTime */);
}
- // TODO: Remove.
- mDictionaryWriter.addBigramWords(word0, word1, frequency, isValid,
- 0 /* lastTouchedTime */);
}
});
}
@@ -303,9 +338,10 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
public void run() {
if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) {
mBinaryDictionary.removeBigramWords(word0, word1);
+ } else {
+ // TODO: Remove.
+ mDictionaryWriter.removeBigramWords(word0, word1);
}
- // TODO: Remove.
- mDictionaryWriter.removeBigramWords(word0, word1);
}
});
}
@@ -322,26 +358,39 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
getExecutor(mFilename).executePrioritized(new Runnable() {
@Override
public void run() {
- final ArrayList<SuggestedWordInfo> inMemDictSuggestion = composer.isBatchMode() ?
- null : mDictionaryWriter.getSuggestionsWithSessionId(composer, prevWord,
- proximityInfo, blockOffensiveWords, additionalFeaturesOptions,
- sessionId);
- // TODO: Remove checking mIsUpdatable and use native suggestion.
- if (mBinaryDictionary != null && !mIsUpdatable) {
+ if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) {
+ if (mBinaryDictionary == null) {
+ holder.set(null);
+ return;
+ }
final ArrayList<SuggestedWordInfo> binarySuggestion =
mBinaryDictionary.getSuggestionsWithSessionId(composer, prevWord,
proximityInfo, blockOffensiveWords, additionalFeaturesOptions,
sessionId);
- if (inMemDictSuggestion == null) {
- holder.set(binarySuggestion);
- } else if (binarySuggestion == null) {
- holder.set(inMemDictSuggestion);
+ holder.set(binarySuggestion);
+ } else {
+ final ArrayList<SuggestedWordInfo> inMemDictSuggestion =
+ composer.isBatchMode() ? null :
+ mDictionaryWriter.getSuggestionsWithSessionId(composer,
+ prevWord, proximityInfo, blockOffensiveWords,
+ additionalFeaturesOptions, sessionId);
+ // TODO: Remove checking mIsUpdatable and use native suggestion.
+ if (mBinaryDictionary != null && !mIsUpdatable) {
+ final ArrayList<SuggestedWordInfo> binarySuggestion =
+ mBinaryDictionary.getSuggestionsWithSessionId(composer, prevWord,
+ proximityInfo, blockOffensiveWords,
+ additionalFeaturesOptions, sessionId);
+ if (inMemDictSuggestion == null) {
+ holder.set(binarySuggestion);
+ } else if (binarySuggestion == null) {
+ holder.set(inMemDictSuggestion);
+ } else {
+ binarySuggestion.addAll(inMemDictSuggestion);
+ holder.set(binarySuggestion);
+ }
} else {
- binarySuggestion.addAll(inMemDictSuggestion);
- holder.set(binarySuggestion);
+ holder.set(inMemDictSuggestion);
}
- } else {
- holder.set(inMemDictSuggestion);
}
}
});
@@ -411,8 +460,9 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
final BinaryDictionary newBinaryDictionary = new BinaryDictionary(filename, 0, length,
true /* useFullEditDistance */, null, mDictType, mIsUpdatable);
- // Ensure all threads accessing the current dictionary have finished before swapping in
- // the new one.
+ // Ensure all threads accessing the current dictionary have finished before
+ // swapping in the new one.
+ // TODO: Ensure multi-thread assignment of mBinaryDictionary.
final BinaryDictionary oldBinaryDictionary = mBinaryDictionary;
getExecutor(mFilename).executePrioritized(new Runnable() {
@Override
@@ -443,8 +493,33 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
if (needsToReloadBeforeWriting()) {
mDictionaryWriter.clear();
loadDictionaryAsync();
+ mDictionaryWriter.write(mFilename);
+ } else {
+ if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) {
+ if (mBinaryDictionary == null || !mBinaryDictionary.isValidDictionary()) {
+ final File file = new File(mContext.getFilesDir(), mFilename);
+ final FusionDictionary dict = new FusionDictionary(new PtNodeArray(),
+ new FusionDictionary.DictionaryOptions(new HashMap<String,String>(),
+ false, false));
+ final DictEncoder dictEncoder = new Ver3DictEncoder(file);
+ try {
+ dictEncoder.writeDictionary(dict, FORMAT_OPTIONS);
+ } catch (IOException e) {
+ Log.e(TAG, "Exception in creating new dictionary file.", e);
+ } catch (UnsupportedFormatException e) {
+ Log.e(TAG, "Exception in creating new dictionary file.", e);
+ }
+ } else {
+ if (mBinaryDictionary.needsToRunGC()) {
+ mBinaryDictionary.flushWithGC();
+ } else {
+ mBinaryDictionary.flush();
+ }
+ }
+ } else {
+ mDictionaryWriter.write(mFilename);
+ }
}
- mDictionaryWriter.write(mFilename);
}
/**
@@ -539,7 +614,9 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
getExecutor(mFilename).executePrioritized(new Runnable() {
@Override
public void run() {
- loadDictionaryAsync();
+ if (!ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) {
+ loadDictionaryAsync();
+ }
}
});
}
@@ -547,7 +624,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
/**
* Generate binary dictionary using DictionaryWriter.
*/
- protected void asyncWriteBinaryDictionary() {
+ protected void asyncFlashAllBinaryDictionary() {
final Runnable newTask = new Runnable() {
@Override
public void run() {
@@ -620,8 +697,12 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
@Override
public void run() {
if (mDictType == Dictionary.TYPE_USER_HISTORY) {
- holder.set(((DynamicPersonalizationDictionaryWriter) mDictionaryWriter)
- .isInDictionaryForTests(word));
+ if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) {
+ holder.set(mBinaryDictionary.isValidWord(word));
+ } else {
+ holder.set(((DynamicPersonalizationDictionaryWriter) mDictionaryWriter)
+ .isInDictionaryForTests(word));
+ }
}
}
});
diff --git a/java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java b/java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java
index 9364fb0349bf57526967e6976994548f8cf3a8b9..075d7e3c3626cced04a5eb7cf772ff5864ee2883 100644
--- a/java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java
+++ b/java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java
@@ -74,12 +74,12 @@ public abstract class DynamicPredictionDictionaryBase extends ExpandableBinaryDi
@Override
public void close() {
- // Close only binary dictionary to reuse this dictionary.
- // super.close();
- closeBinaryDictionary();
+ if (!ExpandableBinaryDictionary.ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) {
+ closeBinaryDictionary();
+ }
// Flush pending writes.
// TODO: Remove after this class become to use a dynamic binary dictionary.
- asyncWriteBinaryDictionary();
+ asyncFlashAllBinaryDictionary();
Settings.writeLastUserHistoryWriteTime(mPrefs, mLocale);
}
@@ -212,6 +212,6 @@ public abstract class DynamicPredictionDictionaryBase extends ExpandableBinaryDi
// Clear the node structure on memory
clear();
// Then flush the cleared state of the dictionary on disk.
- asyncWriteBinaryDictionary();
+ asyncFlashAllBinaryDictionary();
}
}
diff --git a/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java b/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java
index bf44a14244c902fde44b5564788714bb81a5eddb..d605cdb843c0c688eaabce99fd4cc3729b7a0d6d 100644
--- a/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java
+++ b/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java
@@ -100,7 +100,11 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
Thread.sleep(TimeUnit.MILLISECONDS.convert(5L, TimeUnit.SECONDS));
} catch (InterruptedException e) {
}
- for (int i = 0; i < 10 && i < numberOfWords; ++i) {
+ // Limit word count to check when using a Java on memory dictionary.
+ final int wordCountToCheck =
+ ExpandableBinaryDictionary.ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE ?
+ numberOfWords : 10;
+ for (int i = 0; i < wordCountToCheck; ++i) {
final String word = words.get(i);
// This may fail as long as we use tryLock on inserting the bigram words
assertTrue(dict.isInDictionaryForTests(word));
@@ -202,4 +206,41 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
}
}
}
+
+ public void testAddManyWords() {
+ File dictFile = null;
+ final String testFilenameSuffix = "testRandomWords" + System.currentTimeMillis();
+ final int numberOfWords =
+ ExpandableBinaryDictionary.ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE ?
+ 10000 : 1000;
+ final Random random = new Random(123456);
+
+ UserHistoryPredictionDictionary dict =
+ PersonalizationHelper.getUserHistoryPredictionDictionary(getContext(),
+ testFilenameSuffix, mPrefs);
+ try {
+ addAndWriteRandomWords(testFilenameSuffix, numberOfWords, random,
+ true /* checksContents */);
+ dict.close();
+ } finally {
+ try {
+ Log.d(TAG, "waiting for writing ...");
+ dict.shutdownExecutorForTests();
+ while (!dict.isTerminatedForTests()) {
+ Thread.sleep(WAIT_TERMINATING_IN_MILLISECONDS);
+ }
+ } catch (InterruptedException e) {
+ Log.d(TAG, "InterruptedException: ", e);
+ }
+ final String fileName = UserHistoryPredictionDictionary.NAME + "." + testFilenameSuffix
+ + ExpandableBinaryDictionary.DICT_FILE_EXTENSION;
+ dictFile = new File(getContext().getFilesDir(), fileName);
+ if (dictFile != null) {
+ assertTrue(dictFile.exists());
+ assertTrue(dictFile.length() >= MIN_USER_HISTORY_DICTIONARY_FILE_SIZE);
+ dictFile.delete();
+ }
+ }
+ }
+
}