diff --git a/java/src/com/android/inputmethod/latin/BinaryDictionary.java b/java/src/com/android/inputmethod/latin/BinaryDictionary.java
index a41cc6a82b1448554b6a688b73c22958624123a7..db4234c63ea9fe79eaff57a1764204ca3c305fb2 100644
--- a/java/src/com/android/inputmethod/latin/BinaryDictionary.java
+++ b/java/src/com/android/inputmethod/latin/BinaryDictionary.java
@@ -26,6 +26,7 @@ import com.android.inputmethod.latin.settings.NativeSuggestOptions;
import com.android.inputmethod.latin.utils.CollectionUtils;
import com.android.inputmethod.latin.utils.JniUtils;
import com.android.inputmethod.latin.utils.StringUtils;
+import com.android.inputmethod.latin.utils.UnigramProperty;
import java.io.File;
import java.util.ArrayList;
@@ -59,6 +60,19 @@ public final class BinaryDictionary extends Dictionary {
public static final int NOT_A_VALID_TIMESTAMP = -1;
+ // Format to get unigram flags from native side via getUnigramPropertyNative().
+ private static final int FORMAT_UNIGRAM_PROPERTY_OUTPUT_FLAG_COUNT = 4;
+ private static final int FORMAT_UNIGRAM_PROPERTY_IS_NOT_A_WORD_INDEX = 0;
+ private static final int FORMAT_UNIGRAM_PROPERTY_IS_BLACKLISTED_INDEX = 1;
+ private static final int FORMAT_UNIGRAM_PROPERTY_HAS_BIGRAMS_INDEX = 2;
+ private static final int FORMAT_UNIGRAM_PROPERTY_HAS_SHORTCUTS_INDEX = 3;
+
+ // Format to get unigram historical info from native side via getUnigramPropertyNative().
+ private static final int FORMAT_UNIGRAM_PROPERTY_OUTPUT_HISTORICAL_INFO_COUNT = 3;
+ private static final int FORMAT_UNIGRAM_PROPERTY_TIMESTAMP_INDEX = 0;
+ private static final int FORMAT_UNIGRAM_PROPERTY_LEVEL_INDEX = 1;
+ private static final int FORMAT_UNIGRAM_PROPERTY_COUNT_INDEX = 2;
+
private long mNativeDict;
private final Locale mLocale;
private final long mDictSize;
@@ -128,6 +142,10 @@ public final class BinaryDictionary extends Dictionary {
private static native int getFormatVersionNative(long dict);
private static native int getProbabilityNative(long dict, int[] word);
private static native int getBigramProbabilityNative(long dict, int[] word0, int[] word1);
+ private static native void getUnigramPropertyNative(long dict, int[] word,
+ int[] outCodePoints, boolean[] outFlags, int[] outProbability,
+ int[] outHistoricalInfo, ArrayList<int[]> outShortcutTargets,
+ ArrayList<Integer> outShortcutProbabilities);
private static native int getSuggestionsNative(long dict, long proximityInfo,
long traverseSession, int[] xCoordinates, int[] yCoordinates, int[] times,
int[] pointerIds, int[] inputCodePoints, int inputSize, int commitPoint,
@@ -285,6 +303,32 @@ public final class BinaryDictionary extends Dictionary {
return getBigramProbabilityNative(mNativeDict, codePoints0, codePoints1);
}
+ @UsedForTesting
+ public UnigramProperty getUnigramProperty(final String word) {
+ if (TextUtils.isEmpty(word)) {
+ return null;
+ }
+ final int[] codePoints = StringUtils.toCodePointArray(word);
+ final int[] outCodePoints = new int[MAX_WORD_LENGTH];
+ final boolean[] outFlags = new boolean[FORMAT_UNIGRAM_PROPERTY_OUTPUT_FLAG_COUNT];
+ final int[] outProbability = new int[1];
+ final int[] outHistoricalInfo =
+ new int[FORMAT_UNIGRAM_PROPERTY_OUTPUT_HISTORICAL_INFO_COUNT];
+ final ArrayList<int[]> outShortcutTargets = CollectionUtils.newArrayList();
+ final ArrayList<Integer> outShortcutProbabilities = CollectionUtils.newArrayList();
+ getUnigramPropertyNative(mNativeDict, codePoints, outCodePoints, outFlags, outProbability,
+ outHistoricalInfo, outShortcutTargets, outShortcutProbabilities);
+ return new UnigramProperty(codePoints,
+ outFlags[FORMAT_UNIGRAM_PROPERTY_IS_NOT_A_WORD_INDEX],
+ outFlags[FORMAT_UNIGRAM_PROPERTY_IS_BLACKLISTED_INDEX],
+ outFlags[FORMAT_UNIGRAM_PROPERTY_HAS_BIGRAMS_INDEX],
+ outFlags[FORMAT_UNIGRAM_PROPERTY_HAS_SHORTCUTS_INDEX], outProbability[0],
+ outHistoricalInfo[FORMAT_UNIGRAM_PROPERTY_TIMESTAMP_INDEX],
+ outHistoricalInfo[FORMAT_UNIGRAM_PROPERTY_LEVEL_INDEX],
+ outHistoricalInfo[FORMAT_UNIGRAM_PROPERTY_COUNT_INDEX],
+ outShortcutTargets, outShortcutProbabilities);
+ }
+
// Add a unigram entry to binary dictionary with unigram attributes in native code.
public void addUnigramWord(final String word, final int probability,
final String shortcutTarget, final int shortcutProbability, final boolean isNotAWord,
diff --git a/java/src/com/android/inputmethod/latin/utils/UnigramProperty.java b/java/src/com/android/inputmethod/latin/utils/UnigramProperty.java
new file mode 100644
index 0000000000000000000000000000000000000000..4feee4393c6ec790992a54acadfa31a84946cfb5
--- /dev/null
+++ b/java/src/com/android/inputmethod/latin/utils/UnigramProperty.java
@@ -0,0 +1,82 @@
+/*
+ * Copyright (C) 2013 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package com.android.inputmethod.latin.utils;
+
+import com.android.inputmethod.annotations.UsedForTesting;
+import com.android.inputmethod.latin.BinaryDictionary;
+import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;
+
+import java.util.ArrayList;
+
+// This has information that belong to a unigram. This class has some detailed attributes such as
+// historical information but they have to be checked only for testing purpose.
+@UsedForTesting
+public class UnigramProperty {
+ public final String mCodePoints;
+ public final boolean mIsNotAWord;
+ public final boolean mIsBlacklisted;
+ public final boolean mHasBigrams;
+ public final boolean mHasShortcuts;
+ public final int mProbability;
+ // mTimestamp, mLevel and mCount are historical info. These values are depend on the
+ // implementation in native code; thus, we must not use them and have any assumptions about
+ // them except for tests.
+ public final int mTimestamp;
+ public final int mLevel;
+ public final int mCount;
+ public final ArrayList<WeightedString> mShortcutTargets = CollectionUtils.newArrayList();
+
+ private static int getCodePointCount(final int[] codePoints) {
+ for (int i = 0; i < codePoints.length; i++) {
+ if (codePoints[i] == 0) {
+ return i;
+ }
+ }
+ return codePoints.length;
+ }
+
+ // This represents invalid unigram when the probability is BinaryDictionary.NOT_A_PROBABILITY.
+ public UnigramProperty(final int[] codePoints, final boolean isNotAWord,
+ final boolean isBlacklisted, final boolean hasBigram,
+ final boolean hasShortcuts, final int probability, final int timestamp,
+ final int level, final int count, final ArrayList<int[]> shortcutTargets,
+ final ArrayList<Integer> shortcutProbabilities) {
+ mCodePoints = new String(codePoints, 0 /* offset */, getCodePointCount(codePoints));
+ mIsNotAWord = isNotAWord;
+ mIsBlacklisted = isBlacklisted;
+ mHasBigrams = hasBigram;
+ mHasShortcuts = hasShortcuts;
+ mProbability = probability;
+ mTimestamp = timestamp;
+ mLevel = level;
+ mCount = count;
+ final int shortcutTargetCount = shortcutTargets.size();
+ for (int i = 0; i < shortcutTargetCount; i++) {
+ final int[] shortcutTargetCodePointArray = shortcutTargets.get(i);
+ final String shortcutTargetString = new String(shortcutTargetCodePointArray,
+ 0 /* offset */, getCodePointCount(shortcutTargetCodePointArray));
+ mShortcutTargets.add(
+ new WeightedString(shortcutTargetString, shortcutProbabilities.get(i)));
+ }
+ }
+
+ @UsedForTesting
+ public boolean isValid() {
+ return mProbability != BinaryDictionary.NOT_A_PROBABILITY;
+ }
+}
\ No newline at end of file
diff --git a/native/jni/Android.mk b/native/jni/Android.mk
index 52ac333c4898e0adf8a71055cd537b12377fbeeb..f2c6d3becf6eb491e4358b9d38ca79660d40bdff 100644
--- a/native/jni/Android.mk
+++ b/native/jni/Android.mk
@@ -58,7 +58,8 @@ LATIN_IME_CORE_SRC_FILES := \
dictionary.cpp \
digraph_utils.cpp \
error_type_utils.cpp \
- multi_bigram_map.cpp) \
+ multi_bigram_map.cpp \
+ unigram_property.cpp) \
$(addprefix suggest/core/layout/, \
additional_proximity_chars.cpp \
proximity_info.cpp \
diff --git a/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp b/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp
index 71b74b24fd7cf878bb1d2f3bdbcb1c581814af5e..716bda5a782ab631b04cb73be97768624f1152cc 100644
--- a/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp
+++ b/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp
@@ -24,6 +24,7 @@
#include "jni.h"
#include "jni_common.h"
#include "suggest/core/dictionary/dictionary.h"
+#include "suggest/core/dictionary/unigram_property.h"
#include "suggest/core/suggest_options.h"
#include "suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.h"
#include "suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h"
@@ -258,6 +259,21 @@ static jint latinime_BinaryDictionary_getBigramProbability(JNIEnv *env, jclass c
word1Length);
}
+static void latinime_BinaryDictionary_getUnigramProperty(JNIEnv *env, jclass clazz,
+ jlong dict, jintArray word, jintArray outCodePoints, jbooleanArray outFlags,
+ jintArray outProbability, jintArray outHistoricalInfo, jobject outShortcutTargets,
+ jobject outShortcutProbabilities) {
+ Dictionary *dictionary = reinterpret_cast<Dictionary *>(dict);
+ if (!dictionary) return;
+ const jsize wordLength = env->GetArrayLength(word);
+ int wordCodePoints[wordLength];
+ env->GetIntArrayRegion(word, 0, wordLength, wordCodePoints);
+ const UnigramProperty unigramProperty = dictionary->getUnigramProperty(
+ wordCodePoints, wordLength);
+ unigramProperty.outputProperties(env, outCodePoints, outFlags, outProbability,
+ outHistoricalInfo, outShortcutTargets, outShortcutProbabilities);
+}
+
static jfloat latinime_BinaryDictionary_calcNormalizedScore(JNIEnv *env, jclass clazz,
jintArray before, jintArray after, jint score) {
jsize beforeLength = env->GetArrayLength(before);
@@ -333,7 +349,6 @@ static void latinime_BinaryDictionary_removeBigramWords(JNIEnv *env, jclass claz
word1Length);
}
-
// Returns how many language model params are processed.
static int latinime_BinaryDictionary_addMultipleDictionaryEntries(JNIEnv *env, jclass clazz,
jlong dict, jobjectArray languageModelParams, jint startIndex) {
@@ -493,6 +508,11 @@ static const JNINativeMethod sMethods[] = {
const_cast<char *>("(J[I[I)I"),
reinterpret_cast<void *>(latinime_BinaryDictionary_getBigramProbability)
},
+ {
+ const_cast<char *>("getUnigramPropertyNative"),
+ const_cast<char *>("(J[I[I[Z[I[ILjava/util/ArrayList;Ljava/util/ArrayList;)V"),
+ reinterpret_cast<void *>(latinime_BinaryDictionary_getUnigramProperty)
+ },
{
const_cast<char *>("calcNormalizedScoreNative"),
const_cast<char *>("([I[II)F"),
diff --git a/native/jni/src/suggest/core/dictionary/dictionary.cpp b/native/jni/src/suggest/core/dictionary/dictionary.cpp
index 4ee5a5ed3f5881d177ae9f51c6afb3541434627c..e68c0a6d8e8d3ad844ca520a1cb7168eda079aee 100644
--- a/native/jni/src/suggest/core/dictionary/dictionary.cpp
+++ b/native/jni/src/suggest/core/dictionary/dictionary.cpp
@@ -143,6 +143,13 @@ void Dictionary::getProperty(const char *const query, const int queryLength, cha
maxResultLength);
}
+const UnigramProperty Dictionary::getUnigramProperty(const int *const codePoints,
+ const int codePointCount) {
+ TimeKeeper::setCurrentTime();
+ return mDictionaryStructureWithBufferPolicy.get()->getUnigramProperty(
+ codePoints, codePointCount);
+}
+
void Dictionary::logDictionaryInfo(JNIEnv *const env) const {
int dictionaryIdCodePointBuffer[HEADER_ATTRIBUTE_BUFFER_SIZE];
int versionStringCodePointBuffer[HEADER_ATTRIBUTE_BUFFER_SIZE];
diff --git a/native/jni/src/suggest/core/dictionary/dictionary.h b/native/jni/src/suggest/core/dictionary/dictionary.h
index 122e4fc4ff667bdcc6f9674cf2a8b8ead0beebc2..b37b4aa18ecb84ca0cd77dfc4062d8cf601ba2cf 100644
--- a/native/jni/src/suggest/core/dictionary/dictionary.h
+++ b/native/jni/src/suggest/core/dictionary/dictionary.h
@@ -22,6 +22,7 @@
#include "defines.h"
#include "jni.h"
#include "suggest/core/dictionary/bigram_dictionary.h"
+#include "suggest/core/dictionary/unigram_property.h"
#include "suggest/core/policy/dictionary_header_structure_policy.h"
#include "suggest/core/policy/dictionary_structure_with_buffer_policy.h"
#include "suggest/core/suggest_interface.h"
@@ -33,6 +34,7 @@ class DictionaryStructureWithBufferPolicy;
class DicTraverseSession;
class ProximityInfo;
class SuggestOptions;
+class UnigramProperty;
class Dictionary {
public:
@@ -92,6 +94,8 @@ class Dictionary {
void getProperty(const char *const query, const int queryLength, char *const outResult,
const int maxResultLength);
+ const UnigramProperty getUnigramProperty(const int *const codePoints, const int codePointCount);
+
const DictionaryStructureWithBufferPolicy *getDictionaryStructurePolicy() const {
return mDictionaryStructureWithBufferPolicy.get();
}
diff --git a/native/jni/src/suggest/core/dictionary/unigram_property.cpp b/native/jni/src/suggest/core/dictionary/unigram_property.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..16bbb69d8963570ea87f9652370bddef21cd4645
--- /dev/null
+++ b/native/jni/src/suggest/core/dictionary/unigram_property.cpp
@@ -0,0 +1,52 @@
+/*
+ * Copyright (C) 2013 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "suggest/core/dictionary/unigram_property.h"
+
+namespace latinime {
+
+void UnigramProperty::outputProperties(JNIEnv *const env, jintArray outCodePoints,
+ jbooleanArray outFlags, jintArray outProbability, jintArray outHistoricalInfo,
+ jobject outShortcutTargets, jobject outShortcutProbabilities) const {
+ env->SetIntArrayRegion(outCodePoints, 0 /* start */, mCodePointCount, mCodePoints);
+ jboolean flags[] = {mIsNotAWord, mIsBlacklisted, mHasBigrams, mHasShortcuts};
+ env->SetBooleanArrayRegion(outFlags, 0 /* start */, NELEMS(flags), flags);
+ env->SetIntArrayRegion(outProbability, 0 /* start */, 1 /* len */, &mProbability);
+ int historicalInfo[] = {mTimestamp, mLevel, mCount};
+ env->SetIntArrayRegion(outHistoricalInfo, 0 /* start */, NELEMS(historicalInfo),
+ historicalInfo);
+
+ jclass integerClass = env->FindClass("java/lang/Integer");
+ jmethodID intToIntegerConstructorId = env->GetMethodID(integerClass, "<init>", "(I)V");
+ jclass arrayListClass = env->FindClass("java/util/ArrayList");
+ jmethodID addMethodId = env->GetMethodID(arrayListClass, "add", "(Ljava/lang/Object;)Z");
+ const int shortcutTargetCount = mShortcutTargets.size();
+ for (int i = 0; i < shortcutTargetCount; ++i) {
+ jintArray shortcutTargetCodePointArray = env->NewIntArray(mShortcutTargets[i].size());
+ env->SetIntArrayRegion(shortcutTargetCodePointArray, 0 /* start */,
+ mShortcutTargets[i].size(), &mShortcutTargets[i][0]);
+ env->CallVoidMethod(outShortcutTargets, addMethodId, shortcutTargetCodePointArray);
+ env->DeleteLocalRef(shortcutTargetCodePointArray);
+ jobject integerProbability = env->NewObject(integerClass, intToIntegerConstructorId,
+ mShortcutProbabilities[i]);
+ env->CallVoidMethod(outShortcutProbabilities, addMethodId, integerProbability);
+ env->DeleteLocalRef(integerProbability);
+ }
+ env->DeleteLocalRef(integerClass);
+ env->DeleteLocalRef(arrayListClass);
+}
+
+} // namespace latinime
diff --git a/native/jni/src/suggest/core/dictionary/unigram_property.h b/native/jni/src/suggest/core/dictionary/unigram_property.h
new file mode 100644
index 0000000000000000000000000000000000000000..e93093bf4edb7b79006eaeb8c829944d78ff2d05
--- /dev/null
+++ b/native/jni/src/suggest/core/dictionary/unigram_property.h
@@ -0,0 +1,87 @@
+/*
+ * Copyright (C) 2013 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LATINIME_UNIGRAM_PROPERTY_H
+#define LATINIME_UNIGRAM_PROPERTY_H
+
+#include <vector>
+
+#include "defines.h"
+#include "jni.h"
+
+namespace latinime {
+
+// This class is used for returning information belonging to a unigram to java side.
+class UnigramProperty {
+ public:
+ // Invalid unigram.
+ UnigramProperty()
+ : mCodePoints(), mCodePointCount(0), mIsNotAWord(false), mIsBlacklisted(false),
+ mHasBigrams(false), mHasShortcuts(false), mProbability(NOT_A_PROBABILITY),
+ mTimestamp(0), mLevel(0), mCount(0), mShortcutTargets(),
+ mShortcutProbabilities() {}
+
+ UnigramProperty(const UnigramProperty &unigramProperty)
+ : mCodePoints(), mCodePointCount(unigramProperty.mCodePointCount),
+ mIsNotAWord(unigramProperty.mIsNotAWord),
+ mIsBlacklisted(unigramProperty.mIsBlacklisted),
+ mHasBigrams(unigramProperty.mHasBigrams),
+ mHasShortcuts(unigramProperty.mHasShortcuts),
+ mProbability(unigramProperty.mProbability),
+ mTimestamp(unigramProperty.mTimestamp), mLevel(unigramProperty.mLevel),
+ mCount(unigramProperty.mCount), mShortcutTargets(unigramProperty.mShortcutTargets),
+ mShortcutProbabilities(unigramProperty.mShortcutProbabilities) {
+ memcpy(mCodePoints, unigramProperty.mCodePoints, sizeof(mCodePoints));
+ }
+
+ UnigramProperty(const int *const codePoints, const int codePointCount,
+ const bool isNotAWord, const bool isBlacklisted, const bool hasBigrams,
+ const bool hasShortcuts, const int probability, const int timestamp,
+ const int level, const int count,
+ const std::vector<std::vector<int> > *const shortcutTargets,
+ const std::vector<int> *const shortcutProbabilities)
+ : mCodePoints(), mCodePointCount(codePointCount),
+ mIsNotAWord(isNotAWord), mIsBlacklisted(isBlacklisted), mHasBigrams(hasBigrams),
+ mHasShortcuts(hasShortcuts), mProbability(probability), mTimestamp(timestamp),
+ mLevel(level), mCount(count), mShortcutTargets(*shortcutTargets),
+ mShortcutProbabilities(*shortcutProbabilities) {
+ memcpy(mCodePoints, codePoints, sizeof(mCodePoints));
+ }
+
+ void outputProperties(JNIEnv *const env, jintArray outCodePoints, jbooleanArray outFlags,
+ jintArray outProbability, jintArray outHistoricalInfo, jobject outShortcutTargets,
+ jobject outShortcutProbabilities) const;
+
+ private:
+ DISALLOW_ASSIGNMENT_OPERATOR(UnigramProperty);
+
+ int mCodePoints[MAX_WORD_LENGTH];
+ int mCodePointCount;
+ bool mIsNotAWord;
+ bool mIsBlacklisted;
+ bool mHasBigrams;
+ bool mHasShortcuts;
+ int mProbability;
+ // Historical information
+ int mTimestamp;
+ int mLevel;
+ int mCount;
+ // Shortcut
+ std::vector<std::vector<int> > mShortcutTargets;
+ std::vector<int> mShortcutProbabilities;
+};
+} // namespace latinime
+#endif // LATINIME_UNIGRAM_PROPERTY_H
diff --git a/native/jni/src/suggest/core/policy/dictionary_structure_with_buffer_policy.h b/native/jni/src/suggest/core/policy/dictionary_structure_with_buffer_policy.h
index 417d22e0d636a582d2edabe70526fce432169440..c74a4ebbe0e0d599c9af217f0dab4b2c48039344 100644
--- a/native/jni/src/suggest/core/policy/dictionary_structure_with_buffer_policy.h
+++ b/native/jni/src/suggest/core/policy/dictionary_structure_with_buffer_policy.h
@@ -18,6 +18,7 @@
#define LATINIME_DICTIONARY_STRUCTURE_POLICY_H
#include "defines.h"
+#include "suggest/core/dictionary/unigram_property.h"
#include "utils/exclusive_ownership_pointer.h"
namespace latinime {
@@ -90,6 +91,10 @@ class DictionaryStructureWithBufferPolicy {
virtual void getProperty(const char *const query, const int queryLength, char *const outResult,
const int maxResultLength) = 0;
+ // Used for testing.
+ virtual const UnigramProperty getUnigramProperty(const int *const codePonts,
+ const int codePointCount) const = 0;
+
protected:
DictionaryStructureWithBufferPolicy() {}
diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v2/patricia_trie_policy.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v2/patricia_trie_policy.h
index e78c82725b6e2b0b93e8c3bb6b2970166116bac8..2adafd22ba50dc97711e4c157b5a662cc7f56de5 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/structure/v2/patricia_trie_policy.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v2/patricia_trie_policy.h
@@ -123,6 +123,12 @@ class PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
}
}
+ const UnigramProperty getUnigramProperty(const int *const codePoints,
+ const int codePointCount) const {
+ // getUnigramProperty is not supported.
+ return UnigramProperty();
+ }
+
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(PatriciaTriePolicy);
diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp
index 979b8e76d3849b40589a44d4b4226911e5a5178a..d1ba1877c1903bf6957debb88185d8e11c54af50 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp
@@ -16,8 +16,11 @@
#include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h"
+#include <vector>
+
#include "suggest/core/dicnode/dic_node.h"
#include "suggest/core/dicnode/dic_node_vector.h"
+#include "suggest/core/dictionary/unigram_property.h"
#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_helper.h"
#include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.h"
#include "suggest/policyimpl/dictionary/utils/forgetting_curve_utils.h"
@@ -290,4 +293,42 @@ void Ver4PatriciaTriePolicy::getProperty(const char *const query, const int quer
}
}
+const UnigramProperty Ver4PatriciaTriePolicy::getUnigramProperty(const int *const codePoints,
+ const int codePointCount) const {
+ const int ptNodePos = getTerminalPtNodePositionOfWord(codePoints, codePointCount,
+ false /* forceLowerCaseSearch */);
+ if (ptNodePos == NOT_A_DICT_POS) {
+ AKLOGE("fetchUnigramProperty is called for invalid word.");
+ return UnigramProperty();
+ }
+ const PtNodeParams ptNodeParams = mNodeReader.fetchNodeInfoInBufferFromPtNodePos(ptNodePos);
+ const ProbabilityEntry probabilityEntry =
+ mBuffers.get()->getProbabilityDictContent()->getProbabilityEntry(
+ ptNodeParams.getTerminalId());
+ // Fetch shortcut information.
+ std::vector<std::vector<int> > shortcutTargets;
+ std::vector<int> shortcutProbabilities;
+ if (ptNodeParams.hasShortcutTargets()) {
+ int shortcutTarget[MAX_WORD_LENGTH];
+ const ShortcutDictContent *const shortcutDictContent =
+ mBuffers.get()->getShortcutDictContent();
+ bool hasNext = true;
+ int shortcutPos = getShortcutPositionOfPtNode(ptNodePos);
+ while (hasNext) {
+ int shortcutTargetLength = 0;
+ int shortcutProbability = NOT_A_PROBABILITY;
+ shortcutDictContent->getShortcutEntryAndAdvancePosition(MAX_WORD_LENGTH, shortcutTarget,
+ &shortcutTargetLength, &shortcutProbability, &hasNext, &shortcutPos);
+ std::vector<int> target(shortcutTarget, shortcutTarget + shortcutTargetLength);
+ shortcutTargets.push_back(target);
+ shortcutProbabilities.push_back(shortcutProbability);
+ }
+ }
+ return UnigramProperty(ptNodeParams.getCodePoints(), ptNodeParams.getCodePointCount(),
+ ptNodeParams.isNotAWord(), ptNodeParams.isBlacklisted(), ptNodeParams.hasBigrams(),
+ ptNodeParams.hasShortcutTargets(), ptNodeParams.getProbability(),
+ probabilityEntry.getTimeStamp(), probabilityEntry.getLevel(),
+ probabilityEntry.getCount(), &shortcutTargets, &shortcutProbabilities);
+}
+
} // namespace latinime
diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h
index 78f3a553d4432e3c96b05e7e8aa5c38736ef044b..db4e8d21cd3852e3de6a39962d818fb1a38290ef 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h
@@ -107,6 +107,9 @@ class Ver4PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
void getProperty(const char *const query, const int queryLength, char *const outResult,
const int maxResultLength);
+ const UnigramProperty getUnigramProperty(const int *const codePoints,
+ const int codePointCount) const;
+
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(Ver4PatriciaTriePolicy);
diff --git a/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java b/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java
index 9cccdeddaae59f3043153699f0a581a80b8ea629..03a302b8fdc6be48c42a2c0476fca474342ec714 100644
--- a/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java
+++ b/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java
@@ -24,6 +24,7 @@ import android.util.Pair;
import com.android.inputmethod.latin.BinaryDictionary.LanguageModelParam;
import com.android.inputmethod.latin.makedict.CodePointUtils;
import com.android.inputmethod.latin.makedict.FormatSpec;
+import com.android.inputmethod.latin.utils.UnigramProperty;
import java.io.File;
import java.io.IOException;
@@ -824,4 +825,52 @@ public class BinaryDictionaryTests extends AndroidTestCase {
assertEquals(probability, binaryDictionary.getBigramProbability(word0, word1));
}
}
+
+ public void testGetUnigramProperties() {
+ testGetUnigramProperties(4 /* formatVersion */);
+ }
+
+ private void testGetUnigramProperties(final int formatVersion) {
+ final long seed = System.currentTimeMillis();
+ final Random random = new Random(seed);
+ final int ITERATION_COUNT = 1000;
+ final int codePointSetSize = 20;
+ final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random);
+
+ File dictFile = null;
+ try {
+ dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion);
+ } catch (IOException e) {
+ fail("IOException while writing an initial dictionary : " + e);
+ }
+ final BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
+ 0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
+ Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
+
+ final UnigramProperty invalidUnigramProperty =
+ binaryDictionary.getUnigramProperty("dummyWord");
+ assertFalse(invalidUnigramProperty.isValid());
+
+ for (int i = 0; i < ITERATION_COUNT; i++) {
+ final String word = CodePointUtils.generateWord(random, codePointSet);
+ final int unigramProbability = random.nextInt(0xFF);
+ final boolean isNotAWord = random.nextBoolean();
+ final boolean isBlacklisted = random.nextBoolean();
+ // TODO: Add tests for shortcut.
+ // TODO: Add tests for historical info.
+ binaryDictionary.addUnigramWord(word, unigramProbability,
+ null /* shortcutTarget */, BinaryDictionary.NOT_A_PROBABILITY,
+ isNotAWord, isBlacklisted, BinaryDictionary.NOT_A_VALID_TIMESTAMP);
+ final UnigramProperty unigramProperty =
+ binaryDictionary.getUnigramProperty(word);
+ assertEquals(word, unigramProperty.mCodePoints);
+ assertTrue(unigramProperty.isValid());
+ assertEquals(isNotAWord, unigramProperty.mIsNotAWord);
+ assertEquals(isBlacklisted, unigramProperty.mIsBlacklisted);
+ assertEquals(false, unigramProperty.mHasBigrams);
+ assertEquals(false, unigramProperty.mHasShortcuts);
+ assertEquals(unigramProbability, unigramProperty.mProbability);
+ assertTrue(unigramProperty.mShortcutTargets.isEmpty());
+ }
+ }
}