diff --git a/java/src/com/android/inputmethod/latin/BinaryDictionary.java b/java/src/com/android/inputmethod/latin/BinaryDictionary.java
index 999508e92721729b07cbc9e16dda74a3122686e3..1e31c96ec9ba1a1aeb6bc956fed95ede80d5b971 100644
--- a/java/src/com/android/inputmethod/latin/BinaryDictionary.java
+++ b/java/src/com/android/inputmethod/latin/BinaryDictionary.java
@@ -214,8 +214,6 @@ public final class BinaryDictionary extends Dictionary {
private static native void removeBigramWordsNative(long dict, int[] word0, int[] word1);
private static native int addMultipleDictionaryEntriesNative(long dict,
LanguageModelParam[] languageModelParams, int startIndex);
- private static native int calculateProbabilityNative(long dict, int unigramProbability,
- int bigramProbability);
private static native String getPropertyNative(long dict, String query);
private static native boolean isCorruptedNative(long dict);
private static native boolean migrateNative(long dict, String dictFilePath,
@@ -551,12 +549,6 @@ public final class BinaryDictionary extends Dictionary {
return true;
}
- @UsedForTesting
- public int calculateProbability(final int unigramProbability, final int bigramProbability) {
- if (!isValidDictionary()) return NOT_A_PROBABILITY;
- return calculateProbabilityNative(mNativeDict, unigramProbability, bigramProbability);
- }
-
@UsedForTesting
public String getPropertyForTest(final String query) {
if (!isValidDictionary()) return "";
diff --git a/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp b/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp
index 18b78c4df4e4b95c6ac8ef5a7efff2ad3c6c5231..fdb87640969be2e82d3d683703595dcf2a2fd95a 100644
--- a/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp
+++ b/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp
@@ -467,16 +467,6 @@ static int latinime_BinaryDictionary_addMultipleDictionaryEntries(JNIEnv *env, j
return languageModelParamCount;
}
-static int latinime_BinaryDictionary_calculateProbabilityNative(JNIEnv *env, jclass clazz,
- jlong dict, jint unigramProbability, jint bigramProbability) {
- Dictionary *dictionary = reinterpret_cast<Dictionary *>(dict);
- if (!dictionary) {
- return NOT_A_PROBABILITY;
- }
- return dictionary->getDictionaryStructurePolicy()->getProbability(unigramProbability,
- bigramProbability);
-}
-
static jstring latinime_BinaryDictionary_getProperty(JNIEnv *env, jclass clazz, jlong dict,
jstring query) {
Dictionary *dictionary = reinterpret_cast<Dictionary *>(dict);
@@ -669,11 +659,6 @@ static const JNINativeMethod sMethods[] = {
"(J[Lcom/android/inputmethod/latin/utils/LanguageModelParam;I)I"),
reinterpret_cast<void *>(latinime_BinaryDictionary_addMultipleDictionaryEntries)
},
- {
- const_cast<char *>("calculateProbabilityNative"),
- const_cast<char *>("(JII)I"),
- reinterpret_cast<void *>(latinime_BinaryDictionary_calculateProbabilityNative)
- },
{
const_cast<char *>("getPropertyNative"),
const_cast<char *>("(JLjava/lang/String;)Ljava/lang/String;"),
diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.cpp
index 56f19dbae88545c1739951c3b115fa617b36feda..d53922763b58b5741557a2a16ae4b7773bf4a124 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.cpp
@@ -38,8 +38,6 @@ const BigramEntry BigramDictContent::getBigramEntryAndAdvancePosition(
int level = 0;
int count = 0;
if (mHasHistoricalInfo) {
- probability = bigramListBuffer->readUintAndAdvancePosition(
- Ver4DictConstants::PROBABILITY_SIZE, bigramEntryPos);
timestamp = bigramListBuffer->readUintAndAdvancePosition(
Ver4DictConstants::TIME_STAMP_FIELD_SIZE, bigramEntryPos);
level = bigramListBuffer->readUintAndAdvancePosition(
@@ -47,7 +45,8 @@ const BigramEntry BigramDictContent::getBigramEntryAndAdvancePosition(
count = bigramListBuffer->readUintAndAdvancePosition(
Ver4DictConstants::WORD_COUNT_FIELD_SIZE, bigramEntryPos);
} else {
- probability = bigramFlags & Ver4DictConstants::BIGRAM_PROBABILITY_MASK;
+ probability = bigramListBuffer->readUintAndAdvancePosition(
+ Ver4DictConstants::PROBABILITY_SIZE, bigramEntryPos);
}
const int encodedTargetTerminalId = bigramListBuffer->readUintAndAdvancePosition(
Ver4DictConstants::BIGRAM_TARGET_TERMINAL_ID_FIELD_SIZE, bigramEntryPos);
@@ -65,21 +64,13 @@ const BigramEntry BigramDictContent::getBigramEntryAndAdvancePosition(
bool BigramDictContent::writeBigramEntryAndAdvancePosition(
const BigramEntry *const bigramEntryToWrite, int *const entryWritingPos) {
BufferWithExtendableBuffer *const bigramListBuffer = getWritableContentBuffer();
- const int bigramFlags = createAndGetBigramFlags(
- mHasHistoricalInfo ? 0 : bigramEntryToWrite->getProbability(),
- bigramEntryToWrite->hasNext());
+ const int bigramFlags = createAndGetBigramFlags(bigramEntryToWrite->hasNext());
if (!bigramListBuffer->writeUintAndAdvancePosition(bigramFlags,
Ver4DictConstants::BIGRAM_FLAGS_FIELD_SIZE, entryWritingPos)) {
AKLOGE("Cannot write bigram flags. pos: %d, flags: %x", *entryWritingPos, bigramFlags);
return false;
}
if (mHasHistoricalInfo) {
- if (!bigramListBuffer->writeUintAndAdvancePosition(bigramEntryToWrite->getProbability(),
- Ver4DictConstants::PROBABILITY_SIZE, entryWritingPos)) {
- AKLOGE("Cannot write bigram probability. pos: %d, probability: %d", *entryWritingPos,
- bigramEntryToWrite->getProbability());
- return false;
- }
const HistoricalInfo *const historicalInfo = bigramEntryToWrite->getHistoricalInfo();
if (!bigramListBuffer->writeUintAndAdvancePosition(historicalInfo->getTimeStamp(),
Ver4DictConstants::TIME_STAMP_FIELD_SIZE, entryWritingPos)) {
@@ -99,6 +90,13 @@ bool BigramDictContent::writeBigramEntryAndAdvancePosition(
historicalInfo->getCount());
return false;
}
+ } else {
+ if (!bigramListBuffer->writeUintAndAdvancePosition(bigramEntryToWrite->getProbability(),
+ Ver4DictConstants::PROBABILITY_SIZE, entryWritingPos)) {
+ AKLOGE("Cannot write bigram probability. pos: %d, probability: %d", *entryWritingPos,
+ bigramEntryToWrite->getProbability());
+ return false;
+ }
}
const int targetTerminalIdToWrite =
(bigramEntryToWrite->getTargetTerminalId() == Ver4DictConstants::NOT_A_TERMINAL_ID) ?
diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.h
index 944e0f9e2a2d7482e6f06e99f442530051cc2918..b8bdb63a804a27e340cbd570a717c5deb5c9541d 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.h
@@ -95,9 +95,8 @@ class BigramDictContent : public SparseTableDictContent {
private:
DISALLOW_COPY_AND_ASSIGN(BigramDictContent);
- int createAndGetBigramFlags(const int probability, const bool hasNext) const {
- return (probability & Ver4DictConstants::BIGRAM_PROBABILITY_MASK)
- | (hasNext ? Ver4DictConstants::BIGRAM_HAS_NEXT_MASK : 0);
+ int createAndGetBigramFlags(const bool hasNext) const {
+ return hasNext ? Ver4DictConstants::BIGRAM_HAS_NEXT_MASK : 0;
}
bool runGCBigramList(const int bigramListPos,
diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp
index 8373dc5495ba5288d924497987db8775853a6f74..7da9e3072dee668d235238ba8c6f3981a6679197 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp
@@ -115,9 +115,7 @@ int Ver4PatriciaTriePolicy::getProbability(const int unigramProbability,
} else if (bigramProbability == NOT_A_PROBABILITY) {
return ProbabilityUtils::backoff(unigramProbability);
} else {
- // bigramProbability is a bigram probability delta.
- return ProbabilityUtils::computeProbabilityForBigram(unigramProbability,
- bigramProbability);
+ return bigramProbability;
}
}
}
@@ -398,7 +396,7 @@ const WordProperty Ver4PatriciaTriePolicy::getWordProperty(const int *const code
const int probability = bigramEntry.hasHistoricalInfo() ?
ForgettingCurveUtils::decodeProbability(
bigramEntry.getHistoricalInfo(), mHeaderPolicy) :
- getProbability(word1Probability, bigramEntry.getProbability());
+ bigramEntry.getProbability();
bigrams.emplace_back(&word1, probability,
historicalInfo->getTimeStamp(), historicalInfo->getLevel(),
historicalInfo->getCount());
diff --git a/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java b/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java
index cfeed0aa40e6a841a13457c505944425dccf11cf..c87c2a970fa6fc59d14a23c41079ef9151df8d46 100644
--- a/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java
+++ b/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java
@@ -46,6 +46,10 @@ public class BinaryDictionaryTests extends AndroidTestCase {
private static final int[] DICT_FORMAT_VERSIONS =
new int[] { FormatSpec.VERSION4, FormatSpec.VERSION4_DEV };
+ private static boolean canCheckBigramProbability(final int formatVersion) {
+ return formatVersion >= FormatSpec.VERSION4_DEV;
+ }
+
private File createEmptyDictionaryAndGetFile(final String dictId,
final int formatVersion) throws IOException {
if (formatVersion == FormatSpec.VERSION4
@@ -298,8 +302,8 @@ public class BinaryDictionaryTests extends AndroidTestCase {
Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
final int unigramProbability = 100;
- final int bigramProbability = 10;
- final int updatedBigramProbability = 15;
+ final int bigramProbability = 150;
+ final int updatedBigramProbability = 200;
addUnigramWord(binaryDictionary, "aaa", unigramProbability);
addUnigramWord(binaryDictionary, "abb", unigramProbability);
addUnigramWord(binaryDictionary, "bcc", unigramProbability);
@@ -308,25 +312,26 @@ public class BinaryDictionaryTests extends AndroidTestCase {
addBigramWords(binaryDictionary, "abb", "aaa", bigramProbability);
addBigramWords(binaryDictionary, "abb", "bcc", bigramProbability);
- final int probability = binaryDictionary.calculateProbability(unigramProbability,
- bigramProbability);
- assertEquals(true, binaryDictionary.isValidBigram("aaa", "abb"));
- assertEquals(true, binaryDictionary.isValidBigram("aaa", "bcc"));
- assertEquals(true, binaryDictionary.isValidBigram("abb", "aaa"));
- assertEquals(true, binaryDictionary.isValidBigram("abb", "bcc"));
- assertEquals(probability, binaryDictionary.getBigramProbability("aaa", "abb"));
- assertEquals(probability, binaryDictionary.getBigramProbability("aaa", "bcc"));
- assertEquals(probability, binaryDictionary.getBigramProbability("abb", "aaa"));
- assertEquals(probability, binaryDictionary.getBigramProbability("abb", "bcc"));
+ assertTrue(binaryDictionary.isValidBigram("aaa", "abb"));
+ assertTrue(binaryDictionary.isValidBigram("aaa", "bcc"));
+ assertTrue(binaryDictionary.isValidBigram("abb", "aaa"));
+ assertTrue(binaryDictionary.isValidBigram("abb", "bcc"));
+ if (canCheckBigramProbability(formatVersion)) {
+ assertEquals(bigramProbability, binaryDictionary.getBigramProbability("aaa", "abb"));
+ assertEquals(bigramProbability, binaryDictionary.getBigramProbability("aaa", "bcc"));
+ assertEquals(bigramProbability, binaryDictionary.getBigramProbability("abb", "aaa"));
+ assertEquals(bigramProbability, binaryDictionary.getBigramProbability("abb", "bcc"));
+ }
addBigramWords(binaryDictionary, "aaa", "abb", updatedBigramProbability);
- final int updatedProbability = binaryDictionary.calculateProbability(unigramProbability,
- updatedBigramProbability);
- assertEquals(updatedProbability, binaryDictionary.getBigramProbability("aaa", "abb"));
+ if (canCheckBigramProbability(formatVersion)) {
+ assertEquals(updatedBigramProbability,
+ binaryDictionary.getBigramProbability("aaa", "abb"));
+ }
- assertEquals(false, binaryDictionary.isValidBigram("bcc", "aaa"));
- assertEquals(false, binaryDictionary.isValidBigram("bcc", "bbc"));
- assertEquals(false, binaryDictionary.isValidBigram("aaa", "aaa"));
+ assertFalse(binaryDictionary.isValidBigram("bcc", "aaa"));
+ assertFalse(binaryDictionary.isValidBigram("bcc", "bbc"));
+ assertFalse(binaryDictionary.isValidBigram("aaa", "aaa"));
assertEquals(Dictionary.NOT_A_PROBABILITY,
binaryDictionary.getBigramProbability("bcc", "aaa"));
assertEquals(Dictionary.NOT_A_PROBABILITY,
@@ -341,11 +346,18 @@ public class BinaryDictionaryTests extends AndroidTestCase {
addUnigramWord(binaryDictionary, "fgh", unigramProbability);
addUnigramWord(binaryDictionary, "abc", unigramProbability);
addUnigramWord(binaryDictionary, "f", unigramProbability);
- assertEquals(probability, binaryDictionary.getBigramProbability("abcde", "fghij"));
+
+ if (canCheckBigramProbability(formatVersion)) {
+ assertEquals(bigramProbability,
+ binaryDictionary.getBigramProbability("abcde", "fghij"));
+ }
assertEquals(Dictionary.NOT_A_PROBABILITY,
binaryDictionary.getBigramProbability("abcde", "fgh"));
addBigramWords(binaryDictionary, "abcde", "fghij", updatedBigramProbability);
- assertEquals(updatedProbability, binaryDictionary.getBigramProbability("abcde", "fghij"));
+ if (canCheckBigramProbability(formatVersion)) {
+ assertEquals(updatedBigramProbability,
+ binaryDictionary.getBigramProbability("abcde", "fghij"));
+ }
dictFile.delete();
}
@@ -396,18 +408,21 @@ public class BinaryDictionaryTests extends AndroidTestCase {
}
final Pair<String, String> bigram = new Pair<String, String>(word0, word1);
bigramWords.add(bigram);
- final int bigramProbability = random.nextInt(0xF);
+ final int unigramProbability = unigramProbabilities.get(word1);
+ final int bigramProbability =
+ unigramProbability + random.nextInt(0xFF - unigramProbability);
bigramProbabilities.put(bigram, bigramProbability);
addBigramWords(binaryDictionary, word0, word1, bigramProbability);
}
for (final Pair<String, String> bigram : bigramWords) {
- final int unigramProbability = unigramProbabilities.get(bigram.second);
final int bigramProbability = bigramProbabilities.get(bigram);
- final int probability = binaryDictionary.calculateProbability(unigramProbability,
- bigramProbability);
- assertEquals(probability,
- binaryDictionary.getBigramProbability(bigram.first, bigram.second));
+ assertEquals(bigramProbability != Dictionary.NOT_A_PROBABILITY,
+ binaryDictionary.isValidBigram(bigram.first, bigram.second));
+ if (canCheckBigramProbability(formatVersion)) {
+ assertEquals(bigramProbability,
+ binaryDictionary.getBigramProbability(bigram.first, bigram.second));
+ }
}
dictFile.delete();
@@ -430,7 +445,7 @@ public class BinaryDictionaryTests extends AndroidTestCase {
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
final int unigramProbability = 100;
- final int bigramProbability = 10;
+ final int bigramProbability = 150;
addUnigramWord(binaryDictionary, "aaa", unigramProbability);
addUnigramWord(binaryDictionary, "abb", unigramProbability);
addUnigramWord(binaryDictionary, "bcc", unigramProbability);
@@ -439,23 +454,23 @@ public class BinaryDictionaryTests extends AndroidTestCase {
addBigramWords(binaryDictionary, "abb", "aaa", bigramProbability);
addBigramWords(binaryDictionary, "abb", "bcc", bigramProbability);
- assertEquals(true, binaryDictionary.isValidBigram("aaa", "abb"));
- assertEquals(true, binaryDictionary.isValidBigram("aaa", "bcc"));
- assertEquals(true, binaryDictionary.isValidBigram("abb", "aaa"));
- assertEquals(true, binaryDictionary.isValidBigram("abb", "bcc"));
+ assertTrue(binaryDictionary.isValidBigram("aaa", "abb"));
+ assertTrue(binaryDictionary.isValidBigram("aaa", "bcc"));
+ assertTrue(binaryDictionary.isValidBigram("abb", "aaa"));
+ assertTrue(binaryDictionary.isValidBigram("abb", "bcc"));
binaryDictionary.removeBigramWords("aaa", "abb");
- assertEquals(false, binaryDictionary.isValidBigram("aaa", "abb"));
+ assertFalse(binaryDictionary.isValidBigram("aaa", "abb"));
addBigramWords(binaryDictionary, "aaa", "abb", bigramProbability);
- assertEquals(true, binaryDictionary.isValidBigram("aaa", "abb"));
+ assertTrue(binaryDictionary.isValidBigram("aaa", "abb"));
binaryDictionary.removeBigramWords("aaa", "bcc");
- assertEquals(false, binaryDictionary.isValidBigram("aaa", "bcc"));
+ assertFalse(binaryDictionary.isValidBigram("aaa", "bcc"));
binaryDictionary.removeBigramWords("abb", "aaa");
- assertEquals(false, binaryDictionary.isValidBigram("abb", "aaa"));
+ assertFalse(binaryDictionary.isValidBigram("abb", "aaa"));
binaryDictionary.removeBigramWords("abb", "bcc");
- assertEquals(false, binaryDictionary.isValidBigram("abb", "bcc"));
+ assertFalse(binaryDictionary.isValidBigram("abb", "bcc"));
binaryDictionary.removeBigramWords("aaa", "abb");
// Test remove non-existing bigram operation.
@@ -537,7 +552,7 @@ public class BinaryDictionaryTests extends AndroidTestCase {
Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
final int unigramProbability = 100;
- final int bigramProbability = 10;
+ final int bigramProbability = 150;
addUnigramWord(binaryDictionary, "aaa", unigramProbability);
addUnigramWord(binaryDictionary, "abb", unigramProbability);
addUnigramWord(binaryDictionary, "bcc", unigramProbability);
@@ -551,18 +566,18 @@ public class BinaryDictionaryTests extends AndroidTestCase {
binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
- final int probability = binaryDictionary.calculateProbability(unigramProbability,
- bigramProbability);
assertEquals(unigramProbability, binaryDictionary.getFrequency("aaa"));
assertEquals(unigramProbability, binaryDictionary.getFrequency("abb"));
assertEquals(unigramProbability, binaryDictionary.getFrequency("bcc"));
- assertEquals(probability, binaryDictionary.getBigramProbability("aaa", "abb"));
- assertEquals(probability, binaryDictionary.getBigramProbability("aaa", "bcc"));
- assertEquals(probability, binaryDictionary.getBigramProbability("abb", "aaa"));
- assertEquals(probability, binaryDictionary.getBigramProbability("abb", "bcc"));
- assertEquals(false, binaryDictionary.isValidBigram("bcc", "aaa"));
- assertEquals(false, binaryDictionary.isValidBigram("bcc", "bbc"));
- assertEquals(false, binaryDictionary.isValidBigram("aaa", "aaa"));
+ if (canCheckBigramProbability(formatVersion)) {
+ assertEquals(bigramProbability, binaryDictionary.getBigramProbability("aaa", "abb"));
+ assertEquals(bigramProbability, binaryDictionary.getBigramProbability("aaa", "bcc"));
+ assertEquals(bigramProbability, binaryDictionary.getBigramProbability("abb", "aaa"));
+ assertEquals(bigramProbability, binaryDictionary.getBigramProbability("abb", "bcc"));
+ }
+ assertFalse(binaryDictionary.isValidBigram("bcc", "aaa"));
+ assertFalse(binaryDictionary.isValidBigram("bcc", "bbc"));
+ assertFalse(binaryDictionary.isValidBigram("aaa", "aaa"));
binaryDictionary.flushWithGC();
binaryDictionary.close();
@@ -617,7 +632,9 @@ public class BinaryDictionaryTests extends AndroidTestCase {
}
final Pair<String, String> bigram = new Pair<String, String>(word0, word1);
bigramWords.add(bigram);
- final int bigramProbability = random.nextInt(0xF);
+ final int unigramProbability = unigramProbabilities.get(word1);
+ final int bigramProbability =
+ unigramProbability + random.nextInt(0xFF - unigramProbability);
bigramProbabilities.put(bigram, bigramProbability);
addBigramWords(binaryDictionary, word0, word1, bigramProbability);
}
@@ -628,13 +645,15 @@ public class BinaryDictionaryTests extends AndroidTestCase {
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
+
for (final Pair<String, String> bigram : bigramWords) {
- final int unigramProbability = unigramProbabilities.get(bigram.second);
final int bigramProbability = bigramProbabilities.get(bigram);
- final int probability = binaryDictionary.calculateProbability(unigramProbability,
- bigramProbability);
- assertEquals(probability,
- binaryDictionary.getBigramProbability(bigram.first, bigram.second));
+ assertEquals(bigramProbability != Dictionary.NOT_A_PROBABILITY,
+ binaryDictionary.isValidBigram(bigram.first, bigram.second));
+ if (canCheckBigramProbability(formatVersion)) {
+ assertEquals(bigramProbability,
+ binaryDictionary.getBigramProbability(bigram.first, bigram.second));
+ }
}
dictFile.delete();
@@ -709,7 +728,9 @@ public class BinaryDictionaryTests extends AndroidTestCase {
if (TextUtils.equals(word0, word1)) {
continue;
}
- final int bigramProbability = random.nextInt(0xF);
+ final int unigramProbability = unigramProbabilities.get(word1);
+ final int bigramProbability =
+ unigramProbability + random.nextInt(0xFF - unigramProbability);
final Pair<String, String> bigram = new Pair<String, String>(word0, word1);
bigramWords.add(bigram);
bigramProbabilities.put(bigram, bigramProbability);
@@ -734,17 +755,20 @@ public class BinaryDictionaryTests extends AndroidTestCase {
// Test whether the all bigram operations are collectlly handled.
for (int i = 0; i < bigramWords.size(); i++) {
final Pair<String, String> bigram = bigramWords.get(i);
- final int unigramProbability = unigramProbabilities.get(bigram.second);
final int probability;
if (bigramProbabilities.containsKey(bigram)) {
final int bigramProbability = bigramProbabilities.get(bigram);
- probability = binaryDictionary.calculateProbability(unigramProbability,
- bigramProbability);
+ probability = bigramProbability;
} else {
probability = Dictionary.NOT_A_PROBABILITY;
}
- assertEquals(probability,
- binaryDictionary.getBigramProbability(bigram.first, bigram.second));
+
+ if (canCheckBigramProbability(formatVersion)) {
+ assertEquals(probability,
+ binaryDictionary.getBigramProbability(bigram.first, bigram.second));
+ }
+ assertEquals(probability != Dictionary.NOT_A_PROBABILITY,
+ binaryDictionary.isValidBigram(bigram.first, bigram.second));
}
binaryDictionary.flushWithGC();
binaryDictionary.close();
@@ -894,7 +918,7 @@ public class BinaryDictionaryTests extends AndroidTestCase {
for (int i = 0; i < languageModelParams.length; i++) {
final String word = CodePointUtils.generateWord(random, codePointSet);
final int probability = random.nextInt(0xFF);
- final int bigramProbability = random.nextInt(0xF);
+ final int bigramProbability = probability + random.nextInt(0xFF - probability);
unigramProbabilities.put(word, probability);
if (prevWord == null) {
languageModelParams[i] = new LanguageModelParam(word, probability,
@@ -920,11 +944,13 @@ public class BinaryDictionaryTests extends AndroidTestCase {
for (Map.Entry<Pair<String, String>, Integer> entry : bigramProbabilities.entrySet()) {
final String word0 = entry.getKey().first;
final String word1 = entry.getKey().second;
- final int unigramProbability = unigramProbabilities.get(word1);
final int bigramProbability = entry.getValue();
- final int probability = binaryDictionary.calculateProbability(
- unigramProbability, bigramProbability);
- assertEquals(probability, binaryDictionary.getBigramProbability(word0, word1));
+ assertEquals(bigramProbability != Dictionary.NOT_A_PROBABILITY,
+ binaryDictionary.isValidBigram(word0, word1));
+ if (canCheckBigramProbability(formatVersion)) {
+ assertEquals(bigramProbability,
+ binaryDictionary.getBigramProbability(word0, word1));
+ }
}
}
@@ -994,7 +1020,9 @@ public class BinaryDictionaryTests extends AndroidTestCase {
}
final String word0 = words.get(word0Index);
final String word1 = words.get(word1Index);
- final int bigramProbability = random.nextInt(0xF);
+ final int unigramProbability = wordProbabilities.get(word1);
+ final int bigramProbability =
+ unigramProbability + random.nextInt(0xFF - unigramProbability);
binaryDictionary.addBigramWords(word0, word1, bigramProbability,
BinaryDictionary.NOT_A_VALID_TIMESTAMP);
if (binaryDictionary.needsToRunGC(false /* mindsBlockByGC */)) {
@@ -1019,12 +1047,11 @@ public class BinaryDictionaryTests extends AndroidTestCase {
for (int j = 0; j < wordProperty.mBigrams.size(); j++) {
final String word1 = wordProperty.mBigrams.get(j).mWord;
assertTrue(bigramWord1s.contains(word1));
- final int bigramProbabilityDelta = bigramProbabilities.get(
- new Pair<String, String>(word0, word1));
- final int unigramProbability = wordProbabilities.get(word1);
- final int bigramProbablity = binaryDictionary.calculateProbability(
- unigramProbability, bigramProbabilityDelta);
- assertEquals(wordProperty.mBigrams.get(j).getProbability(), bigramProbablity);
+ if (canCheckBigramProbability(formatVersion)) {
+ final int bigramProbability = bigramProbabilities.get(
+ new Pair<String, String>(word0, word1));
+ assertEquals(bigramProbability, wordProperty.mBigrams.get(j).getProbability());
+ }
}
}
}
@@ -1082,7 +1109,9 @@ public class BinaryDictionaryTests extends AndroidTestCase {
}
final String word0 = words.get(word0Index);
final String word1 = words.get(word1Index);
- final int bigramProbability = random.nextInt(0xF);
+ final int unigramProbability = wordProbabilitiesToCheckLater.get(word1);
+ final int bigramProbability =
+ unigramProbability + random.nextInt(0xFF - unigramProbability);
binaryDictionary.addBigramWords(word0, word1, bigramProbability,
BinaryDictionary.NOT_A_VALID_TIMESTAMP);
if (binaryDictionary.needsToRunGC(false /* mindsBlockByGC */)) {
@@ -1113,12 +1142,11 @@ public class BinaryDictionaryTests extends AndroidTestCase {
for (int j = 0; j < wordProperty.mBigrams.size(); j++) {
final String word1 = wordProperty.mBigrams.get(j).mWord;
assertTrue(bigramWord1s.contains(word1));
- final int unigramProbability = wordProbabilitiesToCheckLater.get(word1);
final Pair<String, String> bigram = new Pair<String, String>(word0, word1);
- final int bigramProbabilityDelta = bigramProbabilitiesToCheckLater.get(bigram);
- final int bigramProbablity = binaryDictionary.calculateProbability(
- unigramProbability, bigramProbabilityDelta);
- assertEquals(wordProperty.mBigrams.get(j).getProbability(), bigramProbablity);
+ if (canCheckBigramProbability(formatVersion)) {
+ final int bigramProbability = bigramProbabilitiesToCheckLater.get(bigram);
+ assertEquals(bigramProbability, wordProperty.mBigrams.get(j).getProbability());
+ }
bigramSet.remove(bigram);
}
token = result.mNextToken;
@@ -1286,7 +1314,7 @@ public class BinaryDictionaryTests extends AndroidTestCase {
final int unigramProbability = 100;
addUnigramWord(binaryDictionary, "aaa", unigramProbability);
addUnigramWord(binaryDictionary, "bbb", unigramProbability);
- final int bigramProbability = 10;
+ final int bigramProbability = 150;
addBigramWords(binaryDictionary, "aaa", "bbb", bigramProbability);
final int shortcutProbability = 10;
binaryDictionary.addUnigramWord("ccc", unigramProbability, "xxx", shortcutProbability,
@@ -1303,7 +1331,9 @@ public class BinaryDictionaryTests extends AndroidTestCase {
assertEquals(toFormatVersion, binaryDictionary.getFormatVersion());
assertEquals(unigramProbability, binaryDictionary.getFrequency("aaa"));
assertEquals(unigramProbability, binaryDictionary.getFrequency("bbb"));
- // TODO: Add tests for bigram frequency when the implementation gets ready.
+ if (canCheckBigramProbability(toFormatVersion)) {
+ assertEquals(bigramProbability, binaryDictionary.getBigramProbability("aaa", "bbb"));
+ }
assertTrue(binaryDictionary.isValidBigram("aaa", "bbb"));
WordProperty wordProperty = binaryDictionary.getWordProperty("ccc");
assertEquals(1, wordProperty.mShortcutTargets.size());
@@ -1362,7 +1392,9 @@ public class BinaryDictionaryTests extends AndroidTestCase {
}
final String word0 = words.get(word0Index);
final String word1 = words.get(word1Index);
- final int bigramProbability = random.nextInt(0xF);
+ final int unigramProbability = unigramProbabilities.get(word1);
+ final int bigramProbability =
+ random.nextInt(0xFF - unigramProbability) + unigramProbability;
binaryDictionary.addBigramWords(word0, word1, bigramProbability,
BinaryDictionary.NOT_A_VALID_TIMESTAMP);
if (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) {
@@ -1381,7 +1413,10 @@ public class BinaryDictionaryTests extends AndroidTestCase {
binaryDictionary.getPropertyForTest(BinaryDictionary.UNIGRAM_COUNT_QUERY)));
for (final Pair<String, String> bigram : bigrams) {
- // TODO: Add tests for bigram frequency when the implementation gets ready.
+ if (canCheckBigramProbability(toFormatVersion)) {
+ assertEquals((int)bigramProbabilities.get(bigram),
+ binaryDictionary.getBigramProbability(bigram.first, bigram.second));
+ }
assertTrue(binaryDictionary.isValidBigram(bigram.first, bigram.second));
}
assertEquals(bigramProbabilities.size(), Integer.parseInt(