From 12d63820d459edd71a46fa2495fea98b3c785f2d Mon Sep 17 00:00:00 2001
From: Jatin Matani <jatinm@google.com>
Date: Thu, 30 Oct 2014 11:33:39 -0700
Subject: [PATCH] Hook for fetching sync content from UserHistoryDict
Add API to ExpandableBinaryDictionary to dump content
from a given dictionary. We use this for dumping data
for sync process.
Refactored UserHistoryDictionaryTests to scrap out the util
methods for testing. These utility methods would be used
for testing sync + user dictionary code in LatinIMEGoogleTests
Bug:18106539
Change-Id: I357f9192ea1bd69a526d0b620c25616a2e8e9d5b
---
.../latin/ExpandableBinaryDictionary.java | 37 +++++
.../PersonalizationHelper.java | 3 +-
.../UserHistoryDictionaryTests.java | 136 ++++-------------
.../UserHistoryDictionaryTestsHelper.java | 144 ++++++++++++++++++
4 files changed, 213 insertions(+), 107 deletions(-)
create mode 100644 tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTestsHelper.java
diff --git a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java
index d9d22e0fc6..0b61dc18bc 100644
--- a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java
+++ b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java
@@ -77,6 +77,9 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
private static final int DICTIONARY_FORMAT_VERSION = FormatSpec.VERSION4;
+ private static final WordProperty[] DEFAULT_WORD_PROPERTIES_FOR_SYNC =
+ new WordProperty[0] /* default */;
+
/** The application context. */
protected final Context mContext;
@@ -802,4 +805,38 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
}
});
}
+
+ /**
+ * Returns dictionary content required for syncing.
+ */
+ public WordProperty[] getWordPropertiesForSyncing() {
+ reloadDictionaryIfRequired();
+ final AsyncResultHolder<WordProperty[]> result = new AsyncResultHolder<>();
+ asyncExecuteTaskWithLock(mLock.readLock(), "sync-read", new Runnable() {
+ @Override
+ public void run() {
+ final ArrayList<WordProperty> wordPropertyList = new ArrayList<>();
+ final BinaryDictionary binaryDictionary = getBinaryDictionary();
+ if (binaryDictionary == null) {
+ return;
+ }
+ int token = 0;
+ do {
+ // TODO: We need a new API that returns *new* un-synced data.
+ final BinaryDictionary.GetNextWordPropertyResult result =
+ binaryDictionary.getNextWordProperty(token);
+ final WordProperty wordProperty = result.mWordProperty;
+ if (wordProperty == null) {
+ break;
+ }
+ wordPropertyList.add(wordProperty);
+ token = result.mNextToken;
+ } while (token != 0);
+ result.set(wordPropertyList.toArray(new WordProperty[wordPropertyList.size()]));
+ }
+ });
+ // TODO: Figure out the best timeout duration for this API.
+ return result.get(DEFAULT_WORD_PROPERTIES_FOR_SYNC,
+ TIMEOUT_FOR_READ_OPS_IN_MILLISECONDS);
+ }
}
diff --git a/java/src/com/android/inputmethod/latin/personalization/PersonalizationHelper.java b/java/src/com/android/inputmethod/latin/personalization/PersonalizationHelper.java
index 8c5eb0aa73..cefb0ddd56 100644
--- a/java/src/com/android/inputmethod/latin/personalization/PersonalizationHelper.java
+++ b/java/src/com/android/inputmethod/latin/personalization/PersonalizationHelper.java
@@ -58,8 +58,7 @@ public class PersonalizationHelper {
final UserHistoryDictionary dict = ref == null ? null : ref.get();
if (dict != null) {
if (DEBUG) {
- Log.d(TAG, "Use cached UserHistoryDictionary for " + locale +
- " & account" + accountName);
+ Log.d(TAG, "Use cached UserHistoryDictionary with lookup: " + lookupStr);
}
dict.reloadDictionaryIfRequired();
return dict;
diff --git a/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java b/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java
index a84df28c94..d83c4a55b5 100644
--- a/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java
+++ b/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java
@@ -25,15 +25,11 @@ import android.util.Log;
import com.android.inputmethod.latin.ExpandableBinaryDictionary;
import com.android.inputmethod.latin.NgramContext;
import com.android.inputmethod.latin.NgramContext.WordInfo;
-import com.android.inputmethod.latin.common.FileUtils;
import com.android.inputmethod.latin.settings.LocalSettingsConstants;
import com.android.inputmethod.latin.utils.BinaryDictionaryUtils;
import com.android.inputmethod.latin.utils.DistracterFilter;
import java.io.File;
-import java.io.FilenameFilter;
-import java.util.ArrayList;
-import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Random;
@@ -48,34 +44,13 @@ import javax.annotation.Nullable;
public class UserHistoryDictionaryTests extends AndroidTestCase {
private static final String TAG = UserHistoryDictionaryTests.class.getSimpleName();
private static final int WAIT_FOR_WRITING_FILE_IN_MILLISECONDS = 3000;
- private static final String TEST_LOCALE_PREFIX = "test_";
private static final String TEST_ACCOUNT = "account@example.com";
- private static final String[] CHARACTERS = {
- "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m",
- "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z"
- };
-
private int mCurrentTime = 0;
private SharedPreferences mPrefs;
private String mLastKnownAccount = null;
- private void removeAllTestDictFiles() {
- final Locale dummyLocale = new Locale(TEST_LOCALE_PREFIX);
- final String dictName = UserHistoryDictionary.getUserHistoryDictName(
- UserHistoryDictionary.NAME, dummyLocale, null /* dictFile */, getContext());
- final File dictFile = ExpandableBinaryDictionary.getDictFile(
- mContext, dictName, null /* dictFile */);
- final FilenameFilter filenameFilter = new FilenameFilter() {
- @Override
- public boolean accept(final File dir, final String filename) {
- return filename.startsWith(UserHistoryDictionary.NAME + "." + TEST_LOCALE_PREFIX);
- }
- };
- FileUtils.deleteFilteredFiles(dictFile.getParentFile(), filenameFilter);
- }
-
private static void printAllFiles(final File dir) {
Log.d(TAG, dir.getAbsolutePath());
for (final File file : dir.listFiles()) {
@@ -83,7 +58,7 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
}
}
- private static void checkExistenceAndRemoveDictFile(final UserHistoryDictionary dict,
+ private static void assertDictionaryExists(final UserHistoryDictionary dict,
final File dictFile) {
Log.d(TAG, "waiting for writing ...");
dict.waitAllTasksForTests();
@@ -97,12 +72,7 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
Log.e(TAG, "Interrupted during waiting for writing the dict file.");
}
}
- assertTrue("check exisiting of " + dictFile, dictFile.exists());
- FileUtils.deleteRecursively(dictFile);
- }
-
- private static Locale getDummyLocale(final String name) {
- return new Locale(TEST_LOCALE_PREFIX + name + System.currentTimeMillis());
+ assertTrue("Following dictionary file doesn't exist: " + dictFile, dictFile.exists());
}
@Override
@@ -115,12 +85,14 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
updateAccountName(TEST_ACCOUNT);
resetCurrentTimeForTestMode();
- removeAllTestDictFiles();
+ UserHistoryDictionaryTestsHelper.removeAllTestDictFiles(
+ UserHistoryDictionaryTestsHelper.TEST_LOCALE_PREFIX, mContext);
}
@Override
protected void tearDown() throws Exception {
- removeAllTestDictFiles();
+ UserHistoryDictionaryTestsHelper.removeAllTestDictFiles(
+ UserHistoryDictionaryTestsHelper.TEST_LOCALE_PREFIX, mContext);
stopTestModeInNativeCode();
// Restore the account that was present before running the test.
@@ -164,58 +136,6 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
return BinaryDictionaryUtils.setCurrentTimeForTest(-1);
}
- /**
- * Generates a random word.
- */
- private static String generateWord(final int value) {
- final int lengthOfChars = CHARACTERS.length;
- final StringBuilder builder = new StringBuilder();
- long lvalue = Math.abs((long)value);
- while (lvalue > 0) {
- builder.append(CHARACTERS[(int)(lvalue % lengthOfChars)]);
- lvalue /= lengthOfChars;
- }
- return builder.toString();
- }
-
- private static List<String> generateWords(final int number, final Random random) {
- final HashSet<String> wordSet = new HashSet<>();
- while (wordSet.size() < number) {
- wordSet.add(generateWord(random.nextInt()));
- }
- return new ArrayList<>(wordSet);
- }
-
- private static void addToDict(final UserHistoryDictionary dict, final List<String> words,
- final int timestamp) {
- NgramContext ngramContext = NgramContext.EMPTY_PREV_WORDS_INFO;
- for (final String word : words) {
- UserHistoryDictionary.addToDictionary(dict, ngramContext, word, true, timestamp,
- DistracterFilter.EMPTY_DISTRACTER_FILTER);
- ngramContext = ngramContext.getNextNgramContext(new WordInfo(word));
- }
- }
-
- /**
- * @param checkContents if true, checks whether written words are actually in the dictionary
- * or not.
- */
- private void addAndWriteRandomWords(final UserHistoryDictionary dict,
- final int numberOfWords, final Random random, final boolean checkContents) {
- final List<String> words = generateWords(numberOfWords, random);
- // Add random words to the user history dictionary.
- addToDict(dict, words, mCurrentTime);
- if (checkContents) {
- dict.waitAllTasksForTests();
- for (int i = 0; i < numberOfWords; ++i) {
- final String word = words.get(i);
- assertTrue(dict.isInDictionary(word));
- }
- }
- // write to file.
- dict.close();
- }
-
/**
* Clear all entries in the user history dictionary.
* @param dict the user history dictionary.
@@ -230,19 +150,19 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
public void testRandomWords() {
Log.d(TAG, "This test can be used for profiling.");
Log.d(TAG, "Usage: please set UserHistoryDictionary.PROFILE_SAVE_RESTORE to true.");
- final Locale dummyLocale = getDummyLocale("random_words");
+ final Locale dummyLocale = UserHistoryDictionaryTestsHelper.getDummyLocale("random_words");
final String dictName = UserHistoryDictionary.getUserHistoryDictName(
UserHistoryDictionary.NAME, dummyLocale, null /* dictFile */, getContext());
final File dictFile = ExpandableBinaryDictionary.getDictFile(
mContext, dictName, null /* dictFile */);
final UserHistoryDictionary dict = PersonalizationHelper.getUserHistoryDictionary(
getContext(), dummyLocale, TEST_ACCOUNT);
-
+ clearHistory(dict);
final int numberOfWords = 1000;
final Random random = new Random(123456);
- clearHistory(dict);
- addAndWriteRandomWords(dict, numberOfWords, random, true /* checksContents */);
- checkExistenceAndRemoveDictFile(dict, dictFile);
+ assertTrue(UserHistoryDictionaryTestsHelper.addAndWriteRandomWords(
+ dict, numberOfWords, random, true /* checksContents */, mCurrentTime));
+ assertDictionaryExists(dict, dictFile);
}
public void testStressTestForSwitchingLanguagesAndAddingWords() {
@@ -258,7 +178,8 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
// Create filename suffixes for this test.
for (int i = 0; i < numberOfLanguages; i++) {
- final Locale dummyLocale = getDummyLocale("switching_languages" + i);
+ final Locale dummyLocale =
+ UserHistoryDictionaryTestsHelper.getDummyLocale("switching_languages" + i);
final String dictName = UserHistoryDictionary.getUserHistoryDictName(
UserHistoryDictionary.NAME, dummyLocale, null /* dictFile */, getContext());
dictFiles[i] = ExpandableBinaryDictionary.getDictFile(
@@ -273,8 +194,11 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
for (int i = 0; i < numberOfLanguageSwitching; i++) {
final int index = i % numberOfLanguages;
// Switch to dicts[index].
- addAndWriteRandomWords(dicts[index], numberOfWordsInsertedForEachLanguageSwitch,
- random, false /* checksContents */);
+ assertTrue(UserHistoryDictionaryTestsHelper.addAndWriteRandomWords(dicts[index],
+ numberOfWordsInsertedForEachLanguageSwitch,
+ random,
+ false /* checksContents */,
+ mCurrentTime));
}
final long end = System.currentTimeMillis();
@@ -282,13 +206,14 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
+ (end - start) + " ms");
} finally {
for (int i = 0; i < numberOfLanguages; i++) {
- checkExistenceAndRemoveDictFile(dicts[i], dictFiles[i]);
+ assertDictionaryExists(dicts[i], dictFiles[i]);
}
}
}
public void testAddManyWords() {
- final Locale dummyLocale = getDummyLocale("many_random_words");
+ final Locale dummyLocale =
+ UserHistoryDictionaryTestsHelper.getDummyLocale("many_random_words");
final String dictName = UserHistoryDictionary.getUserHistoryDictName(
UserHistoryDictionary.NAME, dummyLocale, null /* dictFile */, getContext());
final File dictFile = ExpandableBinaryDictionary.getDictFile(
@@ -298,23 +223,23 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
final UserHistoryDictionary dict = PersonalizationHelper.getUserHistoryDictionary(
getContext(), dummyLocale, TEST_ACCOUNT);
clearHistory(dict);
- try {
- addAndWriteRandomWords(dict, numberOfWords, random, true /* checksContents */);
- } finally {
- checkExistenceAndRemoveDictFile(dict, dictFile);
- }
+ assertTrue(UserHistoryDictionaryTestsHelper.addAndWriteRandomWords(dict,
+ numberOfWords, random, true /* checksContents */, mCurrentTime));
+ assertDictionaryExists(dict, dictFile);
}
public void testDecaying() {
- final Locale dummyLocale = getDummyLocale("decaying");
+ final Locale dummyLocale = UserHistoryDictionaryTestsHelper.getDummyLocale("decaying");
final UserHistoryDictionary dict = PersonalizationHelper.getUserHistoryDictionary(
getContext(), dummyLocale, TEST_ACCOUNT);
- final int numberOfWords = 5000;
- final Random random = new Random(123456);
resetCurrentTimeForTestMode();
clearHistory(dict);
- final List<String> words = generateWords(numberOfWords, random);
dict.waitAllTasksForTests();
+
+ final int numberOfWords = 5000;
+ final Random random = new Random(123456);
+ final List<String> words = UserHistoryDictionaryTestsHelper.generateWords(numberOfWords,
+ random);
NgramContext ngramContext = NgramContext.EMPTY_PREV_WORDS_INFO;
for (final String word : words) {
UserHistoryDictionary.addToDictionary(dict, ngramContext, word, true, mCurrentTime,
@@ -329,6 +254,7 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
for (final String word : words) {
assertTrue(dict.isInDictionary(word));
}
+ // Long term decay results in words removed from the dictionary.
forcePassingLongTime();
dict.runGCIfRequired();
dict.waitAllTasksForTests();
diff --git a/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTestsHelper.java b/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTestsHelper.java
new file mode 100644
index 0000000000..d394c0faad
--- /dev/null
+++ b/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTestsHelper.java
@@ -0,0 +1,144 @@
+/*
+ * Copyright (C) 2014 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.inputmethod.latin.personalization;
+
+import android.content.Context;
+
+import com.android.inputmethod.latin.NgramContext;
+import com.android.inputmethod.latin.NgramContext.WordInfo;
+import com.android.inputmethod.latin.common.FileUtils;
+import com.android.inputmethod.latin.utils.DistracterFilter;
+
+import java.io.File;
+import java.io.FilenameFilter;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Random;
+
+/**
+ * Utility class for helping while running tests involving {@link UserHistoryDictionary}.
+ */
+public class UserHistoryDictionaryTestsHelper {
+
+ /**
+ * Locale prefix for generating dummy locales for tests.
+ */
+ public static final String TEST_LOCALE_PREFIX = "test-";
+
+ /**
+ * Characters for generating random words.
+ */
+ private static final String[] CHARACTERS = {
+ "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m",
+ "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z"
+ };
+
+ /**
+ * Remove all the test dictionary files created for the given locale.
+ */
+ public static void removeAllTestDictFiles(final String filter, final Context context) {
+ final FilenameFilter filenameFilter = new FilenameFilter() {
+ @Override
+ public boolean accept(final File dir, final String filename) {
+ return filename.startsWith(UserHistoryDictionary.NAME + "." + filter);
+ }
+ };
+ FileUtils.deleteFilteredFiles(context.getFilesDir(), filenameFilter);
+ }
+
+ /**
+ * Generates and writes random words to dictionary. Caller can be assured
+ * that the write tasks would be finished; and its success would be reflected
+ * in the returned boolean.
+ *
+ * @param dict {@link UserHistoryDictionary} to which words should be added.
+ * @param numberOfWords number of words to be added.
+ * @param random helps generate random words.
+ * @param checkContents if true, checks whether written words are actually in the dictionary.
+ * @param currentTime timestamp that would be used for adding the words.
+ * @returns true if all words have been written to dictionary successfully.
+ */
+ public static boolean addAndWriteRandomWords(final UserHistoryDictionary dict,
+ final int numberOfWords, final Random random, final boolean checkContents,
+ final int currentTime) {
+ final List<String> words = generateWords(numberOfWords, random);
+ // Add random words to the user history dictionary.
+ addWordsToDictionary(dict, words, currentTime);
+ boolean success = true;
+ if (checkContents) {
+ dict.waitAllTasksForTests();
+ for (int i = 0; i < numberOfWords; ++i) {
+ final String word = words.get(i);
+ if (!dict.isInDictionary(word)) {
+ success = false;
+ break;
+ }
+ }
+ }
+ // write to file.
+ dict.close();
+ dict.waitAllTasksForTests();
+ return success;
+ }
+
+ private static void addWordsToDictionary(final UserHistoryDictionary dict,
+ final List<String> words, final int timestamp) {
+ NgramContext ngramContext = NgramContext.EMPTY_PREV_WORDS_INFO;
+ for (final String word : words) {
+ UserHistoryDictionary.addToDictionary(dict, ngramContext, word, true, timestamp,
+ DistracterFilter.EMPTY_DISTRACTER_FILTER);
+ ngramContext = ngramContext.getNextNgramContext(new WordInfo(word));
+ }
+ }
+
+ /**
+ * Creates unique test locale for using within tests.
+ */
+ public static Locale getDummyLocale(final String name) {
+ return new Locale(TEST_LOCALE_PREFIX + name + System.currentTimeMillis());
+ }
+
+ /**
+ * Generates random words.
+ *
+ * @param numberOfWords number of words to generate.
+ * @param random salt used for generating random words.
+ */
+ public static List<String> generateWords(final int numberOfWords, final Random random) {
+ final HashSet<String> wordSet = new HashSet<>();
+ while (wordSet.size() < numberOfWords) {
+ wordSet.add(generateWord(random.nextInt()));
+ }
+ return new ArrayList<>(wordSet);
+ }
+
+ /**
+ * Generates a random word.
+ */
+ private static String generateWord(final int value) {
+ final int lengthOfChars = CHARACTERS.length;
+ final StringBuilder builder = new StringBuilder();
+ long lvalue = Math.abs((long)value);
+ while (lvalue > 0) {
+ builder.append(CHARACTERS[(int)(lvalue % lengthOfChars)]);
+ lvalue /= lengthOfChars;
+ }
+ return builder.toString();
+ }
+}
--
GitLab