From 166d8c2293eee268c62ecfba65d2d89d35d97540 Mon Sep 17 00:00:00 2001
From: Keisuke Kuroyanagi <ksk@google.com>
Date: Fri, 6 Jun 2014 19:38:44 +0900
Subject: [PATCH] Make Distracter filter use getMaxFrequencyOfExactMatches().
Bug: 13142176
Bug: 15428247
Change-Id: I5c23fbea2851f891f76f19d9da2cb70ae964569b
---
.../latin/DictionaryCollection.java | 14 +-
.../DistracterFilterUsingSuggestion.java | 155 ++----------------
.../latin/DistracterFilterTest.java | 65 ++++++--
3 files changed, 77 insertions(+), 157 deletions(-)
diff --git a/java/src/com/android/inputmethod/latin/DictionaryCollection.java b/java/src/com/android/inputmethod/latin/DictionaryCollection.java
index 53be28139c..53c78fd001 100644
--- a/java/src/com/android/inputmethod/latin/DictionaryCollection.java
+++ b/java/src/com/android/inputmethod/latin/DictionaryCollection.java
@@ -89,9 +89,17 @@ public final class DictionaryCollection extends Dictionary {
int maxFreq = -1;
for (int i = mDictionaries.size() - 1; i >= 0; --i) {
final int tempFreq = mDictionaries.get(i).getFrequency(word);
- if (tempFreq >= maxFreq) {
- maxFreq = tempFreq;
- }
+ maxFreq = Math.max(tempFreq, maxFreq);
+ }
+ return maxFreq;
+ }
+
+ @Override
+ public int getMaxFrequencyOfExactMatches(final String word) {
+ int maxFreq = -1;
+ for (int i = mDictionaries.size() - 1; i >= 0; --i) {
+ final int tempFreq = mDictionaries.get(i).getMaxFrequencyOfExactMatches(word);
+ maxFreq = Math.max(tempFreq, maxFreq);
}
return maxFreq;
}
diff --git a/java/src/com/android/inputmethod/latin/utils/DistracterFilterUsingSuggestion.java b/java/src/com/android/inputmethod/latin/utils/DistracterFilterUsingSuggestion.java
index b9c7f56710..8c3844ed8c 100644
--- a/java/src/com/android/inputmethod/latin/utils/DistracterFilterUsingSuggestion.java
+++ b/java/src/com/android/inputmethod/latin/utils/DistracterFilterUsingSuggestion.java
@@ -16,33 +16,22 @@
package com.android.inputmethod.latin.utils;
-import java.util.HashMap;
import java.util.List;
import java.util.Locale;
-import java.util.Map;
import java.util.concurrent.TimeUnit;
import android.content.Context;
-import android.content.res.Resources;
-import android.text.InputType;
import android.util.Log;
-import android.view.inputmethod.EditorInfo;
import android.view.inputmethod.InputMethodSubtype;
-import com.android.inputmethod.keyboard.Keyboard;
-import com.android.inputmethod.keyboard.KeyboardId;
-import com.android.inputmethod.keyboard.KeyboardLayoutSet;
-import com.android.inputmethod.latin.Constants;
-import com.android.inputmethod.latin.Dictionary;
import com.android.inputmethod.latin.DictionaryFacilitator;
import com.android.inputmethod.latin.PrevWordsInfo;
-import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo;
-import com.android.inputmethod.latin.WordComposer;
/**
* This class is used to prevent distracters being added to personalization
* or user history dictionaries
*/
+// TODO: Rename.
public class DistracterFilterUsingSuggestion implements DistracterFilter {
private static final String TAG = DistracterFilterUsingSuggestion.class.getSimpleName();
private static final boolean DEBUG = false;
@@ -50,10 +39,7 @@ public class DistracterFilterUsingSuggestion implements DistracterFilter {
private static final long TIMEOUT_TO_WAIT_LOADING_DICTIONARIES_IN_SECONDS = 120;
private final Context mContext;
- private final Map<Locale, InputMethodSubtype> mLocaleToSubtypeMap;
- private final Map<Locale, Keyboard> mLocaleToKeyboardMap;
private final DictionaryFacilitator mDictionaryFacilitator;
- private Keyboard mKeyboard;
private final Object mLock = new Object();
/**
@@ -63,10 +49,7 @@ public class DistracterFilterUsingSuggestion implements DistracterFilter {
*/
public DistracterFilterUsingSuggestion(final Context context) {
mContext = context;
- mLocaleToSubtypeMap = new HashMap<>();
- mLocaleToKeyboardMap = new HashMap<>();
mDictionaryFacilitator = new DictionaryFacilitator();
- mKeyboard = null;
}
@Override
@@ -76,94 +59,6 @@ public class DistracterFilterUsingSuggestion implements DistracterFilter {
@Override
public void updateEnabledSubtypes(final List<InputMethodSubtype> enabledSubtypes) {
- final Map<Locale, InputMethodSubtype> newLocaleToSubtypeMap = new HashMap<>();
- if (enabledSubtypes != null) {
- for (final InputMethodSubtype subtype : enabledSubtypes) {
- final Locale locale = SubtypeLocaleUtils.getSubtypeLocale(subtype);
- if (newLocaleToSubtypeMap.containsKey(locale)) {
- // Multiple subtypes are enabled for one locale.
- // TODO: Investigate what we should do for this case.
- continue;
- }
- newLocaleToSubtypeMap.put(locale, subtype);
- }
- }
- if (mLocaleToSubtypeMap.equals(newLocaleToSubtypeMap)) {
- // Enabled subtypes have not been changed.
- return;
- }
- synchronized (mLock) {
- mLocaleToSubtypeMap.clear();
- mLocaleToSubtypeMap.putAll(newLocaleToSubtypeMap);
- mLocaleToKeyboardMap.clear();
- }
- }
-
- private boolean isDistracter(
- final SuggestionResults suggestionResults, final String consideredWord) {
- int perfectMatchProbability = Dictionary.NOT_A_PROBABILITY;
- for (final SuggestedWordInfo suggestedWordInfo : suggestionResults) {
- if (suggestedWordInfo.mWord.equals(consideredWord)) {
- perfectMatchProbability = mDictionaryFacilitator.getFrequency(consideredWord);
- continue;
- }
- // Exact match can include case errors, accent errors, digraph conversions.
- final boolean isExactMatch = suggestedWordInfo.isExactMatch();
- final boolean isExactMatchWithIntentionalOmission =
- suggestedWordInfo.isExactMatchWithIntentionalOmission();
-
- if (DEBUG) {
- final float normalizedScore = BinaryDictionaryUtils.calcNormalizedScore(
- consideredWord, suggestedWordInfo.mWord, suggestedWordInfo.mScore);
- Log.d(TAG, "consideredWord: " + consideredWord);
- Log.d(TAG, "top suggestion: " + suggestedWordInfo.mWord);
- Log.d(TAG, "suggestionScore: " + suggestedWordInfo.mScore);
- Log.d(TAG, "normalizedScore: " + normalizedScore);
- Log.d(TAG, "isExactMatch: " + isExactMatch);
- Log.d(TAG, "isExactMatchWithIntentionalOmission: "
- + isExactMatchWithIntentionalOmission);
- }
- if (perfectMatchProbability != Dictionary.NOT_A_PROBABILITY) {
- final int topNonPerfectProbability = mDictionaryFacilitator.getFrequency(
- suggestedWordInfo.mWord);
- if (DEBUG) {
- Log.d(TAG, "perfectMatchProbability: " + perfectMatchProbability);
- Log.d(TAG, "topNonPerfectProbability: " + topNonPerfectProbability);
- }
- if (perfectMatchProbability > topNonPerfectProbability) {
- return false;
- }
- }
- return isExactMatch || isExactMatchWithIntentionalOmission;
- }
- return false;
- }
-
- private void loadKeyboardForLocale(final Locale newLocale) {
- final Keyboard cachedKeyboard = mLocaleToKeyboardMap.get(newLocale);
- if (cachedKeyboard != null) {
- mKeyboard = cachedKeyboard;
- return;
- }
- final InputMethodSubtype subtype;
- synchronized (mLock) {
- subtype = mLocaleToSubtypeMap.get(newLocale);
- }
- if (subtype == null) {
- return;
- }
- final EditorInfo editorInfo = new EditorInfo();
- editorInfo.inputType = InputType.TYPE_CLASS_TEXT;
- final KeyboardLayoutSet.Builder builder = new KeyboardLayoutSet.Builder(
- mContext, editorInfo);
- final Resources res = mContext.getResources();
- final int keyboardWidth = ResourceUtils.getDefaultKeyboardWidth(res);
- final int keyboardHeight = ResourceUtils.getDefaultKeyboardHeight(res);
- builder.setKeyboardGeometry(keyboardWidth, keyboardHeight);
- builder.setSubtype(subtype);
- builder.setIsSpellChecker(false /* isSpellChecker */);
- final KeyboardLayoutSet layoutSet = builder.build();
- mKeyboard = layoutSet.getKeyboard(KeyboardId.ELEMENT_ALPHABET);
}
private void loadDictionariesForLocale(final Locale newlocale) throws InterruptedException {
@@ -191,12 +86,6 @@ public class DistracterFilterUsingSuggestion implements DistracterFilter {
}
if (!locale.equals(mDictionaryFacilitator.getLocale())) {
synchronized (mLock) {
- if (!mLocaleToSubtypeMap.containsKey(locale)) {
- Log.e(TAG, "Locale " + locale + " is not enabled.");
- // TODO: Investigate what we should do for disabled locales.
- return false;
- }
- loadKeyboardForLocale(locale);
// Reset dictionaries for the locale.
try {
loadDictionariesForLocale(locale);
@@ -207,37 +96,17 @@ public class DistracterFilterUsingSuggestion implements DistracterFilter {
}
}
}
- if (mKeyboard == null) {
- return false;
+ // The tested word is a distracter when there is a word that is exact matched to the tested
+ // word and its probability is higher than the tested word's probability.
+ final int perfectMatchFreq = mDictionaryFacilitator.getFrequency(testedWord);
+ final int exactMatchFreq = mDictionaryFacilitator.getMaxFrequencyOfExactMatches(testedWord);
+ final boolean isDistracter = perfectMatchFreq < exactMatchFreq;
+ if (DEBUG) {
+ Log.d(TAG, "testedWord: " + testedWord);
+ Log.d(TAG, "perfectMatchFreq: " + perfectMatchFreq);
+ Log.d(TAG, "exactMatchFreq: " + exactMatchFreq);
+ Log.d(TAG, "isDistracter: " + isDistracter);
}
- final WordComposer composer = new WordComposer();
- final int[] codePoints = StringUtils.toCodePointArray(testedWord);
- final int[] coordinates = mKeyboard.getCoordinates(codePoints);
- composer.setComposingWord(codePoints, coordinates, PrevWordsInfo.EMPTY_PREV_WORDS_INFO);
-
- final int trailingSingleQuotesCount = StringUtils.getTrailingSingleQuotesCount(testedWord);
- final String consideredWord = trailingSingleQuotesCount > 0 ?
- testedWord.substring(0, testedWord.length() - trailingSingleQuotesCount) :
- testedWord;
- final AsyncResultHolder<Boolean> holder = new AsyncResultHolder<>();
- ExecutorUtils.getExecutor("check distracters").execute(new Runnable() {
- @Override
- public void run() {
- final SuggestionResults suggestionResults =
- mDictionaryFacilitator.getSuggestionResults(
- composer, PrevWordsInfo.EMPTY_PREV_WORDS_INFO,
- mKeyboard.getProximityInfo(), true /* blockOffensiveWords */,
- null /* additionalFeaturesOptions */, 0 /* sessionId */,
- null /* rawSuggestions */);
- if (suggestionResults.isEmpty()) {
- holder.set(false);
- return;
- }
- holder.set(isDistracter(suggestionResults, consideredWord));
- }
- });
- // It's OK to block the distracter filtering, but the dictionary lookup should be done
- // sequentially using ExecutorUtils.
- return holder.get(false /* defaultValue */, Constants.GET_SUGGESTED_WORDS_TIMEOUT);
+ return isDistracter;
}
}
diff --git a/tests/src/com/android/inputmethod/latin/DistracterFilterTest.java b/tests/src/com/android/inputmethod/latin/DistracterFilterTest.java
index 406e9a9b89..b7f2271be1 100644
--- a/tests/src/com/android/inputmethod/latin/DistracterFilterTest.java
+++ b/tests/src/com/android/inputmethod/latin/DistracterFilterTest.java
@@ -42,11 +42,6 @@ public class DistracterFilterTest extends InputTestsBase {
final Locale localeEnUs = new Locale("en", "US");
String typedWord;
- typedWord = "google";
- // For this test case, we consider "google" is a distracter to "Google".
- assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries(
- EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs));
-
typedWord = "Bill";
// For this test case, we consider "Bill" is a distracter to "bill".
assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries(
@@ -83,15 +78,20 @@ public class DistracterFilterTest extends InputTestsBase {
EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs));
typedWord = "cafe";
- // For this test case, we consider "café" is not a distracter to any word in dictionaries.
+ // For this test case, we consider "cafe" is not a distracter to any word in dictionaries.
assertFalse(mDistracterFilter.isDistracterToWordsInDictionaries(
EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs));
- typedWord = "ill";
- // For this test case, we consider "ill" is not a distracter to any word in dictionaries.
+ typedWord = "I'll";
+ // For this test case, we consider "I'll" is not a distracter to any word in dictionaries.
assertFalse(mDistracterFilter.isDistracterToWordsInDictionaries(
EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs));
+ typedWord = "ill";
+ // For this test case, we consider "ill" is a distracter to "I'll"
+ assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries(
+ EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs));
+
typedWord = "asdfd";
// For this test case, we consider "asdfd" is not a distracter to any word in dictionaries.
assertFalse(
@@ -101,8 +101,51 @@ public class DistracterFilterTest extends InputTestsBase {
typedWord = "thank";
// For this test case, we consider "thank" is not a distracter to any other word
// in dictionaries.
- assertFalse(
- mDistracterFilter.isDistracterToWordsInDictionaries(
- EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs));
+ assertFalse(mDistracterFilter.isDistracterToWordsInDictionaries(
+ EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs));
+
+ final Locale localeDeDe = new Locale("de", "DE");
+
+ typedWord = "fuer";
+ // For this test case, we consider "fuer" is a distracter to "für".
+ assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries(
+ EMPTY_PREV_WORDS_INFO, typedWord, localeDeDe));
+
+ typedWord = "fUEr";
+ // For this test case, we consider "fUEr" is a distracter to "für".
+ assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries(
+ EMPTY_PREV_WORDS_INFO, typedWord, localeDeDe));
+
+ typedWord = "fur";
+ // For this test case, we consider "fur" is a distracter to "für".
+ assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries(
+ EMPTY_PREV_WORDS_INFO, typedWord, localeDeDe));
+
+ final Locale localeFrFr = new Locale("fr", "FR");
+
+ typedWord = "a";
+ // For this test case, we consider "a" is a distracter to "Ã ".
+ assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries(
+ EMPTY_PREV_WORDS_INFO, typedWord, localeFrFr));
+
+ typedWord = "Ã ";
+ // For this test case, we consider "Ã " is not a distracter to any word in dictionaries.
+ assertFalse(mDistracterFilter.isDistracterToWordsInDictionaries(
+ EMPTY_PREV_WORDS_INFO, typedWord, localeFrFr));
+
+ typedWord = "etre";
+ // For this test case, we consider "etre" is a distracter to "être".
+ assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries(
+ EMPTY_PREV_WORDS_INFO, typedWord, localeFrFr));
+
+ typedWord = "États-unis";
+ // For this test case, we consider "États-unis" is a distracter to "États-Unis".
+ assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries(
+ EMPTY_PREV_WORDS_INFO, typedWord, localeFrFr));
+
+ typedWord = "ÉtatsUnis";
+ // For this test case, we consider "ÉtatsUnis" is a distracter to "États-Unis".
+ assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries(
+ EMPTY_PREV_WORDS_INFO, typedWord, localeFrFr));
}
}
--
GitLab