diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java
index 369184573adb762904c9a4ceda272d06075e6f8e..9f2345962b4eb2675fac8bb2412656fb1742dd4b 100644
--- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java
+++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java
@@ -511,7 +511,7 @@ public final class BinaryDictDecoderUtils {
final WeightedString word = getWordAtPosition(dictDecoder, headerSize,
bigram.mAddress, options);
final int reconstructedFrequency =
- BinaryDictIOUtils.reconstructBigramFrequency(word.mFrequency,
+ BinaryDictIOUtils.reconstructBigramFrequency(word.getProbability(),
bigram.mFrequency);
bigrams.add(new WeightedString(word.mWord, reconstructedFrequency));
}
@@ -618,7 +618,7 @@ public final class BinaryDictDecoderUtils {
// words that are not also registered as unigrams so we don't have to avoid
// them explicitly here.
for (final WeightedString bigram : w.mBigrams) {
- newDict.setBigram(w.mWord, bigram.mWord, bigram.mFrequency);
+ newDict.setBigram(w.mWord, bigram.mWord, bigram.getProbability());
}
}
}
diff --git a/java/src/com/android/inputmethod/latin/makedict/FusionDictionary.java b/java/src/com/android/inputmethod/latin/makedict/FusionDictionary.java
index 5b0e8399ac728eb1fb2e30a092431f4011dfe22b..ef23acb7145291aa54f0118c07ef640c79a44bc1 100644
--- a/java/src/com/android/inputmethod/latin/makedict/FusionDictionary.java
+++ b/java/src/com/android/inputmethod/latin/makedict/FusionDictionary.java
@@ -67,29 +67,40 @@ public final class FusionDictionary implements Iterable<Word> {
}
/**
- * A string with a frequency.
+ * A string with a probability.
*
* This represents an "attribute", that is either a bigram or a shortcut.
*/
public static final class WeightedString {
public final String mWord;
- public int mFrequency;
- public WeightedString(String word, int frequency) {
+ public ProbabilityInfo mProbabilityInfo;
+
+ public WeightedString(final String word, final int probability) {
mWord = word;
- mFrequency = frequency;
+ mProbabilityInfo = new ProbabilityInfo(probability);
+ }
+
+ public int getProbability() {
+ return mProbabilityInfo.mProbability;
+ }
+
+ public void setProbability(final int probability) {
+ mProbabilityInfo = new ProbabilityInfo(probability);
}
@Override
public int hashCode() {
- return Arrays.hashCode(new Object[] { mWord, mFrequency });
+ return Arrays.hashCode(new Object[] { mWord, mProbabilityInfo.mProbability,
+ mProbabilityInfo.mTimestamp, mProbabilityInfo.mLevel,
+ mProbabilityInfo.mCount });
}
@Override
public boolean equals(Object o) {
if (o == this) return true;
if (!(o instanceof WeightedString)) return false;
- WeightedString w = (WeightedString)o;
- return mWord.equals(w.mWord) && mFrequency == w.mFrequency;
+ final WeightedString w = (WeightedString)o;
+ return mWord.equals(w.mWord) && mProbabilityInfo.equals(w.mProbabilityInfo);
}
}
@@ -200,18 +211,18 @@ public final class FusionDictionary implements Iterable<Word> {
}
/**
- * Adds a word to the bigram list. Updates the frequency if the word already
+ * Adds a word to the bigram list. Updates the probability if the word already
* exists.
*/
- public void addBigram(final String word, final int frequency) {
+ public void addBigram(final String word, final int probability) {
if (mBigrams == null) {
mBigrams = new ArrayList<WeightedString>();
}
WeightedString bigram = getBigram(word);
if (bigram != null) {
- bigram.mFrequency = frequency;
+ bigram.setProbability(probability);
} else {
- bigram = new WeightedString(word, frequency);
+ bigram = new WeightedString(word, probability);
mBigrams.add(bigram);
}
}
@@ -273,8 +284,8 @@ public final class FusionDictionary implements Iterable<Word> {
final WeightedString existingShortcut = getShortcut(shortcut.mWord);
if (existingShortcut == null) {
mShortcutTargets.add(shortcut);
- } else if (existingShortcut.mFrequency < shortcut.mFrequency) {
- existingShortcut.mFrequency = shortcut.mFrequency;
+ } else if (existingShortcut.getProbability() < shortcut.getProbability()) {
+ existingShortcut.setProbability(shortcut.getProbability());
}
}
}
@@ -289,8 +300,8 @@ public final class FusionDictionary implements Iterable<Word> {
final WeightedString existingBigram = getBigram(bigram.mWord);
if (existingBigram == null) {
mBigrams.add(bigram);
- } else if (existingBigram.mFrequency < bigram.mFrequency) {
- existingBigram.mFrequency = bigram.mFrequency;
+ } else if (existingBigram.getProbability() < bigram.getProbability()) {
+ existingBigram.setProbability(bigram.getProbability());
}
}
}
diff --git a/java/src/com/android/inputmethod/latin/makedict/ProbabilityInfo.java b/java/src/com/android/inputmethod/latin/makedict/ProbabilityInfo.java
index c1a43cedfd7c904e4874de6df978e624981f4e3e..79f924cc6b99767f53ef52b31fdda6405e07b74e 100644
--- a/java/src/com/android/inputmethod/latin/makedict/ProbabilityInfo.java
+++ b/java/src/com/android/inputmethod/latin/makedict/ProbabilityInfo.java
@@ -17,6 +17,7 @@
package com.android.inputmethod.latin.makedict;
import com.android.inputmethod.latin.BinaryDictionary;
+import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;
public final class ProbabilityInfo {
public final int mProbability;
@@ -39,8 +40,24 @@ public final class ProbabilityInfo {
mCount = count;
}
+ public boolean hasHistoricalInfo() {
+ return mTimestamp != BinaryDictionary.NOT_A_VALID_TIMESTAMP;
+ }
+
@Override
public String toString() {
return mTimestamp + ":" + mLevel + ":" + mCount;
}
+
+ @Override
+ public boolean equals(Object o) {
+ if (o == this) return true;
+ if (!(o instanceof ProbabilityInfo)) return false;
+ final ProbabilityInfo p = (ProbabilityInfo)o;
+ if (!hasHistoricalInfo() && !p.hasHistoricalInfo()) {
+ return mProbability == p.mProbability;
+ }
+ return mProbability == p.mProbability && mTimestamp == p.mTimestamp && mLevel == p.mLevel
+ && mCount == p.mCount;
+ }
}
\ No newline at end of file
diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver2DictEncoder.java b/java/src/com/android/inputmethod/latin/makedict/Ver2DictEncoder.java
index a3a6c2c34de295398f35ccb727ecdbe05b378b28..3de083ef350d4831aef5e5aa1ba798a567b11f38 100644
--- a/java/src/com/android/inputmethod/latin/makedict/Ver2DictEncoder.java
+++ b/java/src/com/android/inputmethod/latin/makedict/Ver2DictEncoder.java
@@ -197,7 +197,7 @@ public class Ver2DictEncoder implements DictEncoder {
final WeightedString target = shortcutIterator.next();
final int shortcutFlags = BinaryDictEncoderUtils.makeShortcutFlags(
shortcutIterator.hasNext(),
- target.mFrequency);
+ target.getProbability());
mPosition = BinaryDictEncoderUtils.writeUIntToBuffer(mBuffer, mPosition, shortcutFlags,
FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE);
final int shortcutShift = CharEncoding.writeString(mBuffer, mPosition, target.mWord);
@@ -231,7 +231,7 @@ public class Ver2DictEncoder implements DictEncoder {
final int offset = addressOfBigram
- (mPosition + FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE);
final int bigramFlags = BinaryDictEncoderUtils.makeBigramFlags(bigramIterator.hasNext(),
- offset, bigram.mFrequency, unigramFrequencyForThisWord, bigram.mWord);
+ offset, bigram.getProbability(), unigramFrequencyForThisWord, bigram.mWord);
mPosition = BinaryDictEncoderUtils.writeUIntToBuffer(mBuffer, mPosition, bigramFlags,
FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE);
mPosition += BinaryDictEncoderUtils.writeChildrenPosition(mBuffer, mPosition,
diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java b/java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java
index 4b0efbad06b2a0d52a87f9a69ca9bde6dd3bb84c..a5b01358673f3352b5072cbc4c50f47280cae0fb 100644
--- a/java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java
+++ b/java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java
@@ -80,7 +80,7 @@ public class Ver4DictEncoder implements DictEncoder {
} else {
for (final WeightedString shortcutTarget : word.mShortcutTargets) {
binaryDict.addUnigramWord(word.mWord, word.mFrequency,
- shortcutTarget.mWord, shortcutTarget.mFrequency,
+ shortcutTarget.mWord, shortcutTarget.getProbability(),
word.mIsNotAWord, word.mIsBlacklistEntry, 0 /* timestamp */);
}
}
@@ -91,7 +91,7 @@ public class Ver4DictEncoder implements DictEncoder {
for (final Word word0 : dict) {
if (null == word0.mBigrams) continue;
for (final WeightedString word1 : word0.mBigrams) {
- binaryDict.addBigramWords(word0.mWord, word1.mWord, word1.mFrequency,
+ binaryDict.addBigramWords(word0.mWord, word1.mWord, word1.getProbability(),
0 /* timestamp */);
if (binaryDict.needsToRunGC(true /* mindsBlockByGC */)) {
binaryDict.flushWithGC();
diff --git a/java/src/com/android/inputmethod/latin/utils/WordProperty.java b/java/src/com/android/inputmethod/latin/utils/WordProperty.java
index da56b213f088a241e3ee2f9c17b5a28b2a7125d5..37d1102e34c0970eeb98bfc7c513fc3bac3797a1 100644
--- a/java/src/com/android/inputmethod/latin/utils/WordProperty.java
+++ b/java/src/com/android/inputmethod/latin/utils/WordProperty.java
@@ -108,7 +108,7 @@ public class WordProperty {
for (int i = 0; i < mBigramTargets.size(); i++) {
builder.append(" bigram=" + mBigramTargets.get(i).mWord);
builder.append(",");
- builder.append("f=" + mBigramTargets.get(i).mFrequency);
+ builder.append("f=" + mBigramTargets.get(i).getProbability());
if (mBigramProbabilityInfo.get(i).mTimestamp
!= BinaryDictionary.NOT_A_VALID_TIMESTAMP) {
builder.append(",");
@@ -119,7 +119,7 @@ public class WordProperty {
for (int i = 0; i < mShortcutTargets.size(); i++) {
builder.append(" shortcut=" + mShortcutTargets.get(i).mWord);
builder.append(",");
- builder.append("f=" + mShortcutTargets.get(i).mFrequency);
+ builder.append("f=" + mShortcutTargets.get(i).getProbability());
builder.append("\n");
}
return builder.toString();
diff --git a/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java b/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java
index e21975db63a7d2e8284a2453fbba04247feca30a..b1bdef81fa806da690db4af410b687b6663cba40 100644
--- a/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java
+++ b/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java
@@ -962,7 +962,7 @@ public class BinaryDictionaryTests extends AndroidTestCase {
for (int j = 0; j < unigramProperty.mBigramTargets.size(); j++) {
final String word1 = unigramProperty.mBigramTargets.get(j).mWord;
assertTrue(bigramWord1s.contains(word1));
- final int probability = unigramProperty.mBigramTargets.get(j).mFrequency;
+ final int probability = unigramProperty.mBigramTargets.get(j).getProbability();
assertEquals((int)bigramProbabilities.get(new Pair<String, String>(word0, word1)),
probability);
assertEquals(unigramProperty.mBigramProbabilityInfo.get(j).mProbability,
@@ -1053,7 +1053,7 @@ public class BinaryDictionaryTests extends AndroidTestCase {
for (int j = 0; j < wordProperty.mBigramTargets.size(); j++) {
final String word1 = wordProperty.mBigramTargets.get(j).mWord;
assertTrue(bigramWord1s.contains(word1));
- final int probability = wordProperty.mBigramTargets.get(j).mFrequency;
+ final int probability = wordProperty.mBigramTargets.get(j).getProbability();
final Pair<String, String> bigram = new Pair<String, String>(word0, word1);
assertEquals((int)bigramProbabilitiesToCheckLater.get(bigram), probability);
bigramSet.remove(bigram);
@@ -1087,7 +1087,7 @@ public class BinaryDictionaryTests extends AndroidTestCase {
WordProperty wordProperty = binaryDictionary.getWordProperty("aaa");
assertEquals(1, wordProperty.mShortcutTargets.size());
assertEquals("zzz", wordProperty.mShortcutTargets.get(0).mWord);
- assertEquals(shortcutProbability, wordProperty.mShortcutTargets.get(0).mFrequency);
+ assertEquals(shortcutProbability, wordProperty.mShortcutTargets.get(0).getProbability());
final int updatedShortcutProbability = 2;
binaryDictionary.addUnigramWord("aaa", unigramProbability, "zzz",
updatedShortcutProbability, false /* isNotAWord */, false /* isBlacklisted */,
@@ -1096,7 +1096,7 @@ public class BinaryDictionaryTests extends AndroidTestCase {
assertEquals(1, wordProperty.mShortcutTargets.size());
assertEquals("zzz", wordProperty.mShortcutTargets.get(0).mWord);
assertEquals(updatedShortcutProbability,
- wordProperty.mShortcutTargets.get(0).mFrequency);
+ wordProperty.mShortcutTargets.get(0).getProbability());
binaryDictionary.addUnigramWord("aaa", unigramProbability, "yyy",
shortcutProbability, false /* isNotAWord */, false /* isBlacklisted */,
0 /* timestamp */);
@@ -1107,7 +1107,8 @@ public class BinaryDictionaryTests extends AndroidTestCase {
assertEquals(2, wordProperty.mShortcutTargets.size());
for (WeightedString shortcutTarget : wordProperty.mShortcutTargets) {
assertTrue(shortcutTargets.containsKey(shortcutTarget.mWord));
- assertEquals((int)shortcutTargets.get(shortcutTarget.mWord), shortcutTarget.mFrequency);
+ assertEquals((int)shortcutTargets.get(shortcutTarget.mWord),
+ shortcutTarget.getProbability());
shortcutTargets.remove(shortcutTarget.mWord);
}
shortcutTargets.put("zzz", updatedShortcutProbability);
@@ -1117,7 +1118,8 @@ public class BinaryDictionaryTests extends AndroidTestCase {
assertEquals(2, wordProperty.mShortcutTargets.size());
for (WeightedString shortcutTarget : wordProperty.mShortcutTargets) {
assertTrue(shortcutTargets.containsKey(shortcutTarget.mWord));
- assertEquals((int)shortcutTargets.get(shortcutTarget.mWord), shortcutTarget.mFrequency);
+ assertEquals((int)shortcutTargets.get(shortcutTarget.mWord),
+ shortcutTarget.getProbability());
shortcutTargets.remove(shortcutTarget.mWord);
}
}
@@ -1193,7 +1195,7 @@ public class BinaryDictionaryTests extends AndroidTestCase {
for (final WeightedString shortcutTarget : wordProperty.mShortcutTargets) {
final String targetCodePonts = shortcutTarget.mWord;
assertEquals((int)shortcutTargets.get(word).get(targetCodePonts),
- shortcutTarget.mFrequency);
+ shortcutTarget.getProbability());
}
}
}
diff --git a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/CombinedInputOutput.java b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/CombinedInputOutput.java
index 16f82dafdcabb0e4e85ef7f9274390a236fa3665..b9840607ae595e852b21e6d9d9bf0e54ff95d0cf 100644
--- a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/CombinedInputOutput.java
+++ b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/CombinedInputOutput.java
@@ -127,7 +127,7 @@ public class CombinedInputOutput {
if (null != word) {
dict.add(word, freq, shortcuts.isEmpty() ? null : shortcuts, isNotAWord);
for (WeightedString s : bigrams) {
- dict.setBigram(word, s.mWord, s.mFrequency);
+ dict.setBigram(word, s.mWord, s.getProbability());
}
}
if (!shortcuts.isEmpty()) shortcuts = new ArrayList<WeightedString>();
@@ -185,7 +185,7 @@ public class CombinedInputOutput {
if (null != word) {
dict.add(word, freq, shortcuts.isEmpty() ? null : shortcuts, isNotAWord);
for (WeightedString s : bigrams) {
- dict.setBigram(word, s.mWord, s.mFrequency);
+ dict.setBigram(word, s.mWord, s.getProbability());
}
}
@@ -222,13 +222,13 @@ public class CombinedInputOutput {
if (null != word.mShortcutTargets) {
for (WeightedString target : word.mShortcutTargets) {
destination.write(" " + SHORTCUT_TAG + "=" + target.mWord + ","
- + FREQUENCY_TAG + "=" + target.mFrequency + "\n");
+ + FREQUENCY_TAG + "=" + target.getProbability() + "\n");
}
}
if (null != word.mBigrams) {
for (WeightedString bigram : word.mBigrams) {
destination.write(" " + BIGRAM_TAG + "=" + bigram.mWord + ","
- + FREQUENCY_TAG + "=" + bigram.mFrequency + "\n");
+ + FREQUENCY_TAG + "=" + bigram.getProbability() + "\n");
}
}
}
diff --git a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Diff.java b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Diff.java
index 7ac3c67a15dc462b262431ef38de298e6a63c740..c9f6bd50896db2790ebe7dfb36afe533e14dfc8e 100644
--- a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Diff.java
+++ b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Diff.java
@@ -159,7 +159,7 @@ public class Diff extends Dicttool.Command {
if (null == list0) return false;
for (final WeightedString attribute0 : list0) {
System.out.println(type + " removed: " + word + " " + attribute0.mWord + " "
- + attribute0.mFrequency);
+ + attribute0.getProbability());
}
return true;
}
@@ -175,8 +175,8 @@ public class Diff extends Dicttool.Command {
for (final WeightedString attribute1 : list1) {
if (attribute0.mWord.equals(attribute1.mWord)) {
System.out.println(type + " freq changed: " + word + " "
- + attribute0.mWord + " " + attribute0.mFrequency + " -> "
- + attribute1.mFrequency);
+ + attribute0.mWord + " " + attribute0.getProbability() + " -> "
+ + attribute1.getProbability());
list1.remove(attribute1);
foundString = true;
break;
@@ -185,7 +185,7 @@ public class Diff extends Dicttool.Command {
if (!foundString) {
// We come here if we haven't found any matching string.
System.out.println(type + " removed: " + word + " " + attribute0.mWord + " "
- + attribute0.mFrequency);
+ + attribute0.getProbability());
}
} else {
list1.remove(attribute0);
@@ -197,7 +197,7 @@ public class Diff extends Dicttool.Command {
for (final WeightedString attribute1 : list1) {
hasDifferences = true;
System.out.println(type + " added: " + word + " " + attribute1.mWord + " "
- + attribute1.mFrequency);
+ + attribute1.getProbability());
}
return hasDifferences;
}
diff --git a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Info.java b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Info.java
index 350f427725df5306c366456b25507ae3d1ec383b..8f17fcd9409e9596dcf0c3037eccb50629d9b438 100644
--- a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Info.java
+++ b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Info.java
@@ -51,7 +51,8 @@ public class Info extends Dicttool.Command {
if (null != w.mShortcutTargets) {
shortcutCount += w.mShortcutTargets.size();
for (WeightedString shortcutTarget : w.mShortcutTargets) {
- if (FormatSpec.SHORTCUT_WHITELIST_FREQUENCY == shortcutTarget.mFrequency) {
+ if (FormatSpec.SHORTCUT_WHITELIST_FREQUENCY
+ == shortcutTarget.getProbability()) {
++whitelistCount;
}
}
@@ -84,8 +85,9 @@ public class Info extends Dicttool.Command {
} else {
for (final WeightedString shortcutTarget : shortcutTargets) {
System.out.println(" Shortcut target: " + shortcutTarget.mWord + " ("
- + (FormatSpec.SHORTCUT_WHITELIST_FREQUENCY == shortcutTarget.mFrequency
- ? "whitelist" : shortcutTarget.mFrequency) + ")");
+ + (FormatSpec.SHORTCUT_WHITELIST_FREQUENCY
+ == shortcutTarget.getProbability() ?
+ "whitelist" : shortcutTarget.getProbability()) + ")");
}
}
final ArrayList<WeightedString> bigrams = ptNode.getBigrams();
@@ -93,7 +95,8 @@ public class Info extends Dicttool.Command {
System.out.println(" No bigrams");
} else {
for (final WeightedString bigram : bigrams) {
- System.out.println(" Bigram: " + bigram.mWord + " (" + bigram.mFrequency + ")");
+ System.out.println(
+ " Bigram: " + bigram.mWord + " (" + bigram.getProbability() + ")");
}
}
}
diff --git a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/XmlDictInputOutput.java b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/XmlDictInputOutput.java
index d226251c165b2429903b3049b9fe4b8cebe5680a..cdc487b16ee01b358bf37ca594b24155cb259c62 100644
--- a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/XmlDictInputOutput.java
+++ b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/XmlDictInputOutput.java
@@ -319,7 +319,7 @@ public class XmlDictInputOutput {
final ArrayList<WeightedString> bigramList = bigramMap.get(firstWord);
for (final WeightedString bigram : bigramList) {
if (!dict.hasWord(bigram.mWord)) continue;
- dict.setBigram(firstWord, bigram.mWord, bigram.mFrequency);
+ dict.setBigram(firstWord, bigram.mWord, bigram.getProbability());
}
}
return dict;
@@ -369,7 +369,7 @@ public class XmlDictInputOutput {
destination.write("\n");
for (WeightedString target : word.mShortcutTargets) {
destination.write(" <" + SHORTCUT_TAG + " " + FREQUENCY_ATTR + "=\""
- + target.mFrequency + "\">" + target.mWord + "</" + SHORTCUT_TAG
+ + target.getProbability() + "\">" + target.mWord + "</" + SHORTCUT_TAG
+ ">\n");
}
destination.write(" ");
@@ -378,7 +378,8 @@ public class XmlDictInputOutput {
destination.write("\n");
for (WeightedString bigram : word.mBigrams) {
destination.write(" <" + BIGRAM_TAG + " " + FREQUENCY_ATTR + "=\""
- + bigram.mFrequency + "\">" + bigram.mWord + "</" + BIGRAM_TAG + ">\n");
+ + bigram.getProbability() + "\">" + bigram.mWord
+ + "</" + BIGRAM_TAG + ">\n");
}
destination.write(" ");
}