diff --git a/java/src/com/android/inputmethod/latin/BinaryDictionary.java b/java/src/com/android/inputmethod/latin/BinaryDictionary.java
index 95c773855d1d7c39bd751973e03e30efdd573f6f..c7f629f15c9c2545345dd69460aee79230a93dd2 100644
--- a/java/src/com/android/inputmethod/latin/BinaryDictionary.java
+++ b/java/src/com/android/inputmethod/latin/BinaryDictionary.java
@@ -49,11 +49,11 @@ public class BinaryDictionary extends Dictionary {
private int mDicTypeId;
private int mNativeDict;
private int mDictLength;
- private int[] mInputCodes = new int[MAX_WORD_LENGTH * MAX_ALTERNATIVES];
- private char[] mOutputChars = new char[MAX_WORD_LENGTH * MAX_WORDS];
- private char[] mOutputChars_bigrams = new char[MAX_WORD_LENGTH * MAX_BIGRAMS];
- private int[] mFrequencies = new int[MAX_WORDS];
- private int[] mFrequencies_bigrams = new int[MAX_BIGRAMS];
+ private final int[] mInputCodes = new int[MAX_WORD_LENGTH * MAX_ALTERNATIVES];
+ private final char[] mOutputChars = new char[MAX_WORD_LENGTH * MAX_WORDS];
+ private final char[] mOutputChars_bigrams = new char[MAX_WORD_LENGTH * MAX_BIGRAMS];
+ private final int[] mFrequencies = new int[MAX_WORDS];
+ private final int[] mFrequencies_bigrams = new int[MAX_BIGRAMS];
// Keep a reference to the native dict direct buffer in Java to avoid
// unexpected deallocation of the direct buffer.
private ByteBuffer mNativeDictDirectBuffer;
@@ -94,18 +94,19 @@ public class BinaryDictionary extends Dictionary {
}
mDictLength = byteBuffer.capacity();
mNativeDict = openNative(mNativeDictDirectBuffer,
- TYPED_LETTER_MULTIPLIER, FULL_WORD_FREQ_MULTIPLIER);
+ TYPED_LETTER_MULTIPLIER, FULL_WORD_FREQ_MULTIPLIER,
+ MAX_WORD_LENGTH, MAX_WORDS, MAX_ALTERNATIVES);
}
mDicTypeId = dicTypeId;
}
private native int openNative(ByteBuffer bb, int typedLetterMultiplier,
- int fullWordMultiplier);
+ int fullWordMultiplier, int maxWordLength, int maxWords, int maxAlternatives);
private native void closeNative(int dict);
private native boolean isValidWordNative(int nativeData, char[] word, int wordLength);
private native int getSuggestionsNative(int dict, int[] inputCodes, int codesSize,
- char[] outputChars, int[] frequencies, int maxWordLength, int maxWords,
- int maxAlternatives, int skipPos, int[] nextLettersFrequencies, int nextLettersSize);
+ char[] outputChars, int[] frequencies,
+ int[] nextLettersFrequencies, int nextLettersSize);
private native int getBigramsNative(int dict, char[] prevWord, int prevWordLength,
int[] inputCodes, int inputCodesLength, char[] outputChars, int[] frequencies,
int maxWordLength, int maxBigrams, int maxAlternatives);
@@ -131,7 +132,8 @@ public class BinaryDictionary extends Dictionary {
Log.e(TAG, "Read " + got + " bytes, expected " + total);
} else {
mNativeDict = openNative(mNativeDictDirectBuffer,
- TYPED_LETTER_MULTIPLIER, FULL_WORD_FREQ_MULTIPLIER);
+ TYPED_LETTER_MULTIPLIER, FULL_WORD_FREQ_MULTIPLIER,
+ MAX_WORD_LENGTH, MAX_WORDS, MAX_ALTERNATIVES);
mDictLength = total;
}
} catch (IOException e) {
@@ -188,7 +190,7 @@ public class BinaryDictionary extends Dictionary {
final int codesSize = codes.size();
// Won't deal with really long words.
if (codesSize > MAX_WORD_LENGTH - 1) return;
-
+
Arrays.fill(mInputCodes, -1);
for (int i = 0; i < codesSize; i++) {
int[] alternatives = codes.getCodesAt(i);
@@ -199,8 +201,7 @@ public class BinaryDictionary extends Dictionary {
Arrays.fill(mFrequencies, 0);
int count = getSuggestionsNative(mNativeDict, mInputCodes, codesSize, mOutputChars,
- mFrequencies, MAX_WORD_LENGTH, MAX_WORDS, MAX_ALTERNATIVES, -1,
- nextLettersFrequencies,
+ mFrequencies, nextLettersFrequencies,
nextLettersFrequencies != null ? nextLettersFrequencies.length : 0);
for (int j = 0; j < count; j++) {
diff --git a/native/Android.mk b/native/Android.mk
index 97051c92f6a59bea32a7fe51609fb5478793e771..bbbff1efef04577e902bb8d290aa675293070f1a 100644
--- a/native/Android.mk
+++ b/native/Android.mk
@@ -4,9 +4,11 @@ include $(CLEAR_VARS)
LOCAL_C_INCLUDES += $(LOCAL_PATH)/src
LOCAL_SRC_FILES := \
- jni/com_android_inputmethod_latin_BinaryDictionary.cpp \
- src/dictionary.cpp \
- src/char_utils.cpp
+ jni/com_android_inputmethod_latin_BinaryDictionary.cpp \
+ src/bigram_dictionary.cpp \
+ src/char_utils.cpp \
+ src/dictionary.cpp \
+ src/unigram_dictionary.cpp
#FLAG_DBG := true
diff --git a/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp b/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp
index bf7ec0d1afcd78e2bfdb95f442e1bc2fb8506a75..6ed743c2ebc6ecb994ce5104104212f101be1b33 100644
--- a/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp
+++ b/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp
@@ -42,21 +42,23 @@ static void throwException(JNIEnv *env, const char* ex, const char* fmt, int dat
static jint latinime_BinaryDictionary_open
(JNIEnv *env, jobject object, jobject dictDirectBuffer,
- jint typedLetterMultiplier, jint fullWordMultiplier)
+ jint typedLetterMultiplier, jint fullWordMultiplier, jint maxWordLength, jint maxWords,
+ jint maxAlternatives)
{
void *dict = env->GetDirectBufferAddress(dictDirectBuffer);
if (dict == NULL) {
fprintf(stderr, "DICT: Dictionary buffer is null\n");
return 0;
}
- Dictionary *dictionary = new Dictionary(dict, typedLetterMultiplier, fullWordMultiplier);
+ Dictionary *dictionary = new Dictionary(dict, typedLetterMultiplier, fullWordMultiplier,
+ maxWordLength, maxWords, maxAlternatives);
return (jint) dictionary;
}
static int latinime_BinaryDictionary_getSuggestions(
JNIEnv *env, jobject object, jint dict, jintArray inputArray, jint arraySize,
- jcharArray outputArray, jintArray frequencyArray, jint maxWordLength, jint maxWords,
- jint maxAlternatives, jint skipPos, jintArray nextLettersArray, jint nextLettersSize)
+ jcharArray outputArray, jintArray frequencyArray,
+ jintArray nextLettersArray, jint nextLettersSize)
{
Dictionary *dictionary = (Dictionary*) dict;
if (dictionary == NULL) return 0;
@@ -68,8 +70,7 @@ static int latinime_BinaryDictionary_getSuggestions(
: NULL;
int count = dictionary->getSuggestions(inputCodes, arraySize, (unsigned short*) outputChars,
- frequencies, maxWordLength, maxWords, maxAlternatives, skipPos, nextLetters,
- nextLettersSize);
+ frequencies, nextLetters, nextLettersSize);
env->ReleaseIntArrayElements(frequencyArray, frequencies, 0);
env->ReleaseIntArrayElements(inputArray, inputCodes, JNI_ABORT);
@@ -123,17 +124,16 @@ static jboolean latinime_BinaryDictionary_isValidWord
static void latinime_BinaryDictionary_close
(JNIEnv *env, jobject object, jint dict)
{
- Dictionary *dictionary = (Dictionary*) dict;
delete (Dictionary*) dict;
}
// ----------------------------------------------------------------------------
static JNINativeMethod gMethods[] = {
- {"openNative", "(Ljava/nio/ByteBuffer;II)I",
+ {"openNative", "(Ljava/nio/ByteBuffer;IIIII)I",
(void*)latinime_BinaryDictionary_open},
{"closeNative", "(I)V", (void*)latinime_BinaryDictionary_close},
- {"getSuggestionsNative", "(I[II[C[IIIII[II)I", (void*)latinime_BinaryDictionary_getSuggestions},
+ {"getSuggestionsNative", "(I[II[C[I[II)I", (void*)latinime_BinaryDictionary_getSuggestions},
{"isValidWordNative", "(I[CI)Z", (void*)latinime_BinaryDictionary_isValidWord},
{"getBigramsNative", "(I[CI[II[C[IIII)I", (void*)latinime_BinaryDictionary_getBigrams}
};
diff --git a/native/src/bigram_dictionary.cpp b/native/src/bigram_dictionary.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..4d9a612a518fde66ec04405b08b40fc4ea618807
--- /dev/null
+++ b/native/src/bigram_dictionary.cpp
@@ -0,0 +1,31 @@
+/*
+**
+** Copyright 2010, The Android Open Source Project
+**
+** Licensed under the Apache License, Version 2.0 (the "License");
+** you may not use this file except in compliance with the License.
+** You may obtain a copy of the License at
+**
+** http://www.apache.org/licenses/LICENSE-2.0
+**
+** Unless required by applicable law or agreed to in writing, software
+** distributed under the License is distributed on an "AS IS" BASIS,
+** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+** See the License for the specific language governing permissions and
+** limitations under the License.
+*/
+
+#include "bigram_dictionary.h"
+
+namespace latinime {
+
+BigramDictionary::BigramDictionary(void *dict, int typedLetterMultiplier, int fullWordMultiplier,
+ int maxWordLength, int maxWords, int maxAlternatives, Dictionary *parentDictionary)
+{
+}
+
+BigramDictionary::~BigramDictionary()
+{
+}
+// TODO: Move functions related to bigram to here
+} // namespace latinime
diff --git a/native/src/bigram_dictionary.h b/native/src/bigram_dictionary.h
new file mode 100644
index 0000000000000000000000000000000000000000..98146a0e7e44a024276e5851cca27ab46e989d06
--- /dev/null
+++ b/native/src/bigram_dictionary.h
@@ -0,0 +1,32 @@
+/*
+ * Copyright (C) 2010 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LATINIME_BIGRAM_DICTIONARY_H
+#define LATINIME_BIGRAM_DICTIONARY_H
+
+namespace latinime {
+
+class Dictionary;
+class BigramDictionary {
+public:
+ BigramDictionary(void *dict, int typedLetterMultipler, int fullWordMultiplier, int maxWordLength,
+ int maxWords, int maxAlternatives, Dictionary *parentDictionary);
+ ~BigramDictionary();
+private:
+};
+// ----------------------------------------------------------------------------
+}; // namespace latinime
+#endif // LATINIME_BIGRAM_DICTIONARY_H
diff --git a/native/src/dictionary.cpp b/native/src/dictionary.cpp
index b20dc11731bb029037471071db9db97bbf439a4e..a21b80a48165c6fefd7aaf3cd1723e6c4a331d22 100644
--- a/native/src/dictionary.cpp
+++ b/native/src/dictionary.cpp
@@ -16,618 +16,23 @@
*/
#include <stdio.h>
-#include <fcntl.h>
-#include <sys/mman.h>
-#include <string.h>
-
-#ifdef FLAG_DBG
-#define LOG_TAG "LatinIME: dictionary.cpp"
-#include <cutils/log.h>
-#define DEBUG_DICT 1
-#else // FLAG_DBG
-#define LOGI
-#define DEBUG_DICT 0
-#endif // FLAG_DBG
#include "dictionary.h"
-#include "basechars.h"
-#include "char_utils.h"
-
-#define DICTIONARY_VERSION_MIN 200
-#define DICTIONARY_HEADER_SIZE 2
-#define NOT_VALID_WORD -99
-
-#define SUGGEST_MISSING_CHARACTERS true
-#define SUGGEST_MISSING_CHARACTERS_THRESHOLD 5
-
namespace latinime {
-Dictionary::Dictionary(void *dict, int typedLetterMultiplier, int fullWordMultiplier)
+Dictionary::Dictionary(void *dict, int typedLetterMultiplier, int fullWordMultiplier,
+ int maxWordLength, int maxWords, int maxAlternatives)
{
- LOGI("Dictionary - constructor");
- mDict = (unsigned char*) dict;
- mTypedLetterMultiplier = typedLetterMultiplier;
- mFullWordMultiplier = fullWordMultiplier;
- getVersionNumber();
+ mUnigramDictionary = new UnigramDictionary(dict, typedLetterMultiplier, fullWordMultiplier,
+ maxWordLength, maxWords, maxAlternatives, this);
+ mBigramDictionary = new BigramDictionary(dict, typedLetterMultiplier, fullWordMultiplier,
+ maxWordLength, maxWords, maxAlternatives, this);
}
Dictionary::~Dictionary()
{
+ delete mUnigramDictionary;
+ delete mBigramDictionary;
}
-
-int Dictionary::getSuggestions(int *codes, int codesSize, unsigned short *outWords, int *frequencies,
- int maxWordLength, int maxWords, int maxAlternatives, int skipPos,
- int *nextLetters, int nextLettersSize)
-{
-
- initSuggestions(codes, codesSize, outWords, frequencies, maxWordLength, maxWords,
- maxAlternatives);
-
- int suggestedWordsCount = getSuggestionCandidates(codesSize, maxWords, skipPos, nextLetters,
- nextLettersSize);
-
- // If there aren't sufficient suggestions, search for words by allowing wild cards at
- // the different character positions. This feature is not ready for prime-time as we need
- // to figure out the best ranking for such words compared to proximity corrections and
- // completions.
- if (SUGGEST_MISSING_CHARACTERS && suggestedWordsCount < SUGGEST_MISSING_CHARACTERS_THRESHOLD) {
- for (int i = 0; i < codesSize; ++i) {
- int tempCount = getSuggestionCandidates(codesSize, maxWords, i, NULL, 0);
- if (tempCount > suggestedWordsCount) {
- suggestedWordsCount = tempCount;
- break;
- }
- }
- }
-
- if (DEBUG_DICT) {
- LOGI("Returning %d words", suggestedWordsCount);
- LOGI("Next letters: ");
- for (int k = 0; k < nextLettersSize; k++) {
- if (nextLetters[k] > 0) {
- LOGI("%c = %d,", k, nextLetters[k]);
- }
- }
- LOGI("\n");
- }
- return suggestedWordsCount;
-}
-
-void Dictionary::initSuggestions(int *codes, int codesSize, unsigned short *outWords,
- int *frequencies, int maxWordLength, int maxWords, int maxAlternatives) {
- mFrequencies = frequencies;
- mOutputChars = outWords;
- mInputCodes = codes;
- mInputLength = codesSize;
- mMaxAlternatives = maxAlternatives;
- mMaxWordLength = maxWordLength;
- mMaxWords = maxWords;
- mMaxEditDistance = mInputLength < 5 ? 2 : mInputLength / 2;
-}
-
-int Dictionary::getSuggestionCandidates(int inputLength, int maxWords, int skipPos,
- int *nextLetters, int nextLettersSize) {
- if (checkIfDictVersionIsLatest()) {
- getWordsRec(DICTIONARY_HEADER_SIZE, 0, inputLength * 3, false, 1, 0, 0, skipPos,
- nextLetters, nextLettersSize);
- } else {
- getWordsRec(0, 0, inputLength * 3, false, 1, 0, 0, skipPos, nextLetters, nextLettersSize);
- }
-
- // Get the word count
- int suggestedWordsCount = 0;
- while (suggestedWordsCount < maxWords && mFrequencies[suggestedWordsCount] > 0) {
- suggestedWordsCount++;
- }
- return suggestedWordsCount;
-}
-
-void Dictionary::registerNextLetter(unsigned short c, int *nextLetters, int nextLettersSize) {
- if (c < nextLettersSize) {
- nextLetters[c]++;
- }
-}
-
-void
-Dictionary::getVersionNumber()
-{
- mVersion = (mDict[0] & 0xFF);
- mBigram = (mDict[1] & 0xFF);
- LOGI("IN NATIVE SUGGEST Version: %d Bigram : %d \n", mVersion, mBigram);
-}
-
-// Checks whether it has the latest dictionary or the old dictionary
-bool
-Dictionary::checkIfDictVersionIsLatest()
-{
- return (mVersion >= DICTIONARY_VERSION_MIN) && (mBigram == 1 || mBigram == 0);
-}
-
-unsigned short
-Dictionary::getChar(int *pos)
-{
- unsigned short ch = (unsigned short) (mDict[(*pos)++] & 0xFF);
- // If the code is 255, then actual 16 bit code follows (in big endian)
- if (ch == 0xFF) {
- ch = ((mDict[*pos] & 0xFF) << 8) | (mDict[*pos + 1] & 0xFF);
- (*pos) += 2;
- }
- return ch;
-}
-
-int
-Dictionary::getAddress(int *pos)
-{
- int address = 0;
- if ((mDict[*pos] & FLAG_ADDRESS_MASK) == 0) {
- *pos += 1;
- } else {
- address += (mDict[*pos] & (ADDRESS_MASK >> 16)) << 16;
- address += (mDict[*pos + 1] & 0xFF) << 8;
- address += (mDict[*pos + 2] & 0xFF);
- *pos += 3;
- }
- return address;
-}
-
-int
-Dictionary::getFreq(int *pos)
-{
- int freq = mDict[(*pos)++] & 0xFF;
-
- if (checkIfDictVersionIsLatest()) {
- // skipping bigram
- int bigramExist = (mDict[*pos] & FLAG_BIGRAM_READ);
- if (bigramExist > 0) {
- int nextBigramExist = 1;
- while (nextBigramExist > 0) {
- (*pos) += 3;
- nextBigramExist = (mDict[(*pos)++] & FLAG_BIGRAM_CONTINUED);
- }
- } else {
- (*pos)++;
- }
- }
-
- return freq;
-}
-
-int
-Dictionary::wideStrLen(unsigned short *str)
-{
- if (!str) return 0;
- unsigned short *end = str;
- while (*end)
- end++;
- return end - str;
-}
-
-bool
-Dictionary::addWord(unsigned short *word, int length, int frequency)
-{
- word[length] = 0;
- if (DEBUG_DICT) {
- char s[length + 1];
- for (int i = 0; i <= length; i++) s[i] = word[i];
- LOGI("Found word = %s, freq = %d : \n", s, frequency);
- }
-
- // Find the right insertion point
- int insertAt = 0;
- while (insertAt < mMaxWords) {
- if (frequency > mFrequencies[insertAt]
- || (mFrequencies[insertAt] == frequency
- && length < wideStrLen(mOutputChars + insertAt * mMaxWordLength))) {
- break;
- }
- insertAt++;
- }
- if (insertAt < mMaxWords) {
- memmove((char*) mFrequencies + (insertAt + 1) * sizeof(mFrequencies[0]),
- (char*) mFrequencies + insertAt * sizeof(mFrequencies[0]),
- (mMaxWords - insertAt - 1) * sizeof(mFrequencies[0]));
- mFrequencies[insertAt] = frequency;
- memmove((char*) mOutputChars + (insertAt + 1) * mMaxWordLength * sizeof(short),
- (char*) mOutputChars + (insertAt ) * mMaxWordLength * sizeof(short),
- (mMaxWords - insertAt - 1) * sizeof(short) * mMaxWordLength);
- unsigned short *dest = mOutputChars + (insertAt ) * mMaxWordLength;
- while (length--) {
- *dest++ = *word++;
- }
- *dest = 0; // NULL terminate
- if (DEBUG_DICT) LOGI("Added word at %d\n", insertAt);
- return true;
- }
- return false;
-}
-
-bool
-Dictionary::addWordBigram(unsigned short *word, int length, int frequency)
-{
- word[length] = 0;
- if (DEBUG_DICT) {
- char s[length + 1];
- for (int i = 0; i <= length; i++) s[i] = word[i];
- LOGI("Bigram: Found word = %s, freq = %d : \n", s, frequency);
- }
-
- // Find the right insertion point
- int insertAt = 0;
- while (insertAt < mMaxBigrams) {
- if (frequency > mBigramFreq[insertAt]
- || (mBigramFreq[insertAt] == frequency
- && length < wideStrLen(mBigramChars + insertAt * mMaxWordLength))) {
- break;
- }
- insertAt++;
- }
- LOGI("Bigram: InsertAt -> %d maxBigrams: %d\n", insertAt, mMaxBigrams);
- if (insertAt < mMaxBigrams) {
- memmove((char*) mBigramFreq + (insertAt + 1) * sizeof(mBigramFreq[0]),
- (char*) mBigramFreq + insertAt * sizeof(mBigramFreq[0]),
- (mMaxBigrams - insertAt - 1) * sizeof(mBigramFreq[0]));
- mBigramFreq[insertAt] = frequency;
- memmove((char*) mBigramChars + (insertAt + 1) * mMaxWordLength * sizeof(short),
- (char*) mBigramChars + (insertAt ) * mMaxWordLength * sizeof(short),
- (mMaxBigrams - insertAt - 1) * sizeof(short) * mMaxWordLength);
- unsigned short *dest = mBigramChars + (insertAt ) * mMaxWordLength;
- while (length--) {
- *dest++ = *word++;
- }
- *dest = 0; // NULL terminate
- if (DEBUG_DICT) LOGI("Bigram: Added word at %d\n", insertAt);
- return true;
- }
- return false;
-}
-
-unsigned short
-Dictionary::toLowerCase(unsigned short c) {
- if (c < sizeof(BASE_CHARS) / sizeof(BASE_CHARS[0])) {
- c = BASE_CHARS[c];
- }
- if (c >='A' && c <= 'Z') {
- c |= 32;
- } else if (c > 127) {
- c = latin_tolower(c);
- }
- return c;
-}
-
-bool
-Dictionary::sameAsTyped(unsigned short *word, int length)
-{
- if (length != mInputLength) {
- return false;
- }
- int *inputCodes = mInputCodes;
- while (length--) {
- if ((unsigned int) *inputCodes != (unsigned int) *word) {
- return false;
- }
- inputCodes += mMaxAlternatives;
- word++;
- }
- return true;
-}
-
-static char QUOTE = '\'';
-
-void
-Dictionary::getWordsRec(int pos, int depth, int maxDepth, bool completion, int snr, int inputIndex,
- int diffs, int skipPos, int *nextLetters, int nextLettersSize)
-{
- // Optimization: Prune out words that are too long compared to how much was typed.
- if (depth > maxDepth) {
- return;
- }
- if (diffs > mMaxEditDistance) {
- return;
- }
- int count = getCount(&pos);
- int *currentChars = NULL;
- if (mInputLength <= inputIndex) {
- completion = true;
- } else {
- currentChars = mInputCodes + (inputIndex * mMaxAlternatives);
- }
-
- for (int i = 0; i < count; i++) {
- // -- at char
- unsigned short c = getChar(&pos);
- // -- at flag/add
- unsigned short lowerC = toLowerCase(c);
- bool terminal = getTerminal(&pos);
- int childrenAddress = getAddress(&pos);
- // -- after address or flag
- int freq = 1;
- if (terminal) freq = getFreq(&pos);
- // -- after add or freq
-
- // If we are only doing completions, no need to look at the typed characters.
- if (completion) {
- mWord[depth] = c;
- if (terminal) {
- addWord(mWord, depth + 1, freq * snr);
- if (depth >= mInputLength && skipPos < 0) {
- registerNextLetter(mWord[mInputLength], nextLetters, nextLettersSize);
- }
- }
- if (childrenAddress != 0) {
- getWordsRec(childrenAddress, depth + 1, maxDepth, completion, snr, inputIndex,
- diffs, skipPos, nextLetters, nextLettersSize);
- }
- } else if ((c == QUOTE && currentChars[0] != QUOTE) || skipPos == depth) {
- // Skip the ' or other letter and continue deeper
- mWord[depth] = c;
- if (childrenAddress != 0) {
- getWordsRec(childrenAddress, depth + 1, maxDepth, false, snr, inputIndex, diffs,
- skipPos, nextLetters, nextLettersSize);
- }
- } else {
- int j = 0;
- while (currentChars[j] > 0) {
- if (currentChars[j] == lowerC || currentChars[j] == c) {
- int addedWeight = j == 0 ? mTypedLetterMultiplier : 1;
- mWord[depth] = c;
- if (mInputLength == inputIndex + 1) {
- if (terminal) {
- if (//INCLUDE_TYPED_WORD_IF_VALID ||
- !sameAsTyped(mWord, depth + 1)) {
- int finalFreq = freq * snr * addedWeight;
- if (skipPos < 0) finalFreq *= mFullWordMultiplier;
- addWord(mWord, depth + 1, finalFreq);
- }
- }
- if (childrenAddress != 0) {
- getWordsRec(childrenAddress, depth + 1,
- maxDepth, true, snr * addedWeight, inputIndex + 1,
- diffs + (j > 0), skipPos, nextLetters, nextLettersSize);
- }
- } else if (childrenAddress != 0) {
- getWordsRec(childrenAddress, depth + 1, maxDepth,
- false, snr * addedWeight, inputIndex + 1, diffs + (j > 0),
- skipPos, nextLetters, nextLettersSize);
- }
- }
- j++;
- if (skipPos >= 0) break;
- }
- }
- }
-}
-
-int
-Dictionary::getBigramAddress(int *pos, bool advance)
-{
- int address = 0;
-
- address += (mDict[*pos] & 0x3F) << 16;
- address += (mDict[*pos + 1] & 0xFF) << 8;
- address += (mDict[*pos + 2] & 0xFF);
-
- if (advance) {
- *pos += 3;
- }
-
- return address;
-}
-
-int
-Dictionary::getBigramFreq(int *pos)
-{
- int freq = mDict[(*pos)++] & FLAG_BIGRAM_FREQ;
-
- return freq;
-}
-
-
-int
-Dictionary::getBigrams(unsigned short *prevWord, int prevWordLength, int *codes, int codesSize,
- unsigned short *bigramChars, int *bigramFreq, int maxWordLength, int maxBigrams,
- int maxAlternatives)
-{
- mBigramFreq = bigramFreq;
- mBigramChars = bigramChars;
- mInputCodes = codes;
- mInputLength = codesSize;
- mMaxWordLength = maxWordLength;
- mMaxBigrams = maxBigrams;
- mMaxAlternatives = maxAlternatives;
-
- if (mBigram == 1 && checkIfDictVersionIsLatest()) {
- int pos = isValidWordRec(DICTIONARY_HEADER_SIZE, prevWord, 0, prevWordLength);
- LOGI("Pos -> %d\n", pos);
- if (pos < 0) {
- return 0;
- }
-
- int bigramCount = 0;
- int bigramExist = (mDict[pos] & FLAG_BIGRAM_READ);
- if (bigramExist > 0) {
- int nextBigramExist = 1;
- while (nextBigramExist > 0 && bigramCount < maxBigrams) {
- int bigramAddress = getBigramAddress(&pos, true);
- int frequency = (FLAG_BIGRAM_FREQ & mDict[pos]);
- // search for all bigrams and store them
- searchForTerminalNode(bigramAddress, frequency);
- nextBigramExist = (mDict[pos++] & FLAG_BIGRAM_CONTINUED);
- bigramCount++;
- }
- }
-
- return bigramCount;
- }
- return 0;
-}
-
-void
-Dictionary::searchForTerminalNode(int addressLookingFor, int frequency)
-{
- // track word with such address and store it in an array
- unsigned short word[mMaxWordLength];
-
- int pos;
- int followDownBranchAddress = DICTIONARY_HEADER_SIZE;
- bool found = false;
- char followingChar = ' ';
- int depth = -1;
-
- while(!found) {
- bool followDownAddressSearchStop = false;
- bool firstAddress = true;
- bool haveToSearchAll = true;
-
- if (depth >= 0) {
- word[depth] = (unsigned short) followingChar;
- }
- pos = followDownBranchAddress; // pos start at count
- int count = mDict[pos] & 0xFF;
- LOGI("count - %d\n",count);
- pos++;
- for (int i = 0; i < count; i++) {
- // pos at data
- pos++;
- // pos now at flag
- if (!getFirstBitOfByte(&pos)) { // non-terminal
- if (!followDownAddressSearchStop) {
- int addr = getBigramAddress(&pos, false);
- if (addr > addressLookingFor) {
- followDownAddressSearchStop = true;
- if (firstAddress) {
- firstAddress = false;
- haveToSearchAll = true;
- } else if (!haveToSearchAll) {
- break;
- }
- } else {
- followDownBranchAddress = addr;
- followingChar = (char)(0xFF & mDict[pos-1]);
- if (firstAddress) {
- firstAddress = false;
- haveToSearchAll = false;
- }
- }
- }
- pos += 3;
- } else if (getFirstBitOfByte(&pos)) { // terminal
- if (addressLookingFor == (pos-1)) { // found !!
- depth++;
- word[depth] = (0xFF & mDict[pos-1]);
- found = true;
- break;
- }
- if (getSecondBitOfByte(&pos)) { // address + freq (4 byte)
- if (!followDownAddressSearchStop) {
- int addr = getBigramAddress(&pos, false);
- if (addr > addressLookingFor) {
- followDownAddressSearchStop = true;
- if (firstAddress) {
- firstAddress = false;
- haveToSearchAll = true;
- } else if (!haveToSearchAll) {
- break;
- }
- } else {
- followDownBranchAddress = addr;
- followingChar = (char)(0xFF & mDict[pos-1]);
- if (firstAddress) {
- firstAddress = false;
- haveToSearchAll = true;
- }
- }
- }
- pos += 4;
- } else { // freq only (2 byte)
- pos += 2;
- }
-
- // skipping bigram
- int bigramExist = (mDict[pos] & FLAG_BIGRAM_READ);
- if (bigramExist > 0) {
- int nextBigramExist = 1;
- while (nextBigramExist > 0) {
- pos += 3;
- nextBigramExist = (mDict[pos++] & FLAG_BIGRAM_CONTINUED);
- }
- } else {
- pos++;
- }
- }
- }
- depth++;
- if (followDownBranchAddress == 0) {
- LOGI("ERROR!!! Cannot find bigram!!");
- break;
- }
- }
- if (checkFirstCharacter(word)) {
- addWordBigram(word, depth, frequency);
- }
-}
-
-bool
-Dictionary::checkFirstCharacter(unsigned short *word)
-{
- // Checks whether this word starts with same character or neighboring characters of
- // what user typed.
-
- int *inputCodes = mInputCodes;
- int maxAlt = mMaxAlternatives;
- while (maxAlt > 0) {
- if ((unsigned int) *inputCodes == (unsigned int) *word) {
- return true;
- }
- inputCodes++;
- maxAlt--;
- }
- return false;
-}
-
-bool
-Dictionary::isValidWord(unsigned short *word, int length)
-{
- if (checkIfDictVersionIsLatest()) {
- return (isValidWordRec(DICTIONARY_HEADER_SIZE, word, 0, length) != NOT_VALID_WORD);
- } else {
- return (isValidWordRec(0, word, 0, length) != NOT_VALID_WORD);
- }
-}
-
-int
-Dictionary::isValidWordRec(int pos, unsigned short *word, int offset, int length) {
- // returns address of bigram data of that word
- // return -99 if not found
-
- int count = getCount(&pos);
- unsigned short currentChar = (unsigned short) word[offset];
- for (int j = 0; j < count; j++) {
- unsigned short c = getChar(&pos);
- int terminal = getTerminal(&pos);
- int childPos = getAddress(&pos);
- if (c == currentChar) {
- if (offset == length - 1) {
- if (terminal) {
- return (pos+1);
- }
- } else {
- if (childPos != 0) {
- int t = isValidWordRec(childPos, word, offset + 1, length);
- if (t > 0) {
- return t;
- }
- }
- }
- }
- if (terminal) {
- getFreq(&pos);
- }
- // There could be two instances of each alphabet - upper and lower case. So continue
- // looking ...
- }
- return NOT_VALID_WORD;
-}
-
-
} // namespace latinime
diff --git a/native/src/dictionary.h b/native/src/dictionary.h
index 2dac3b2c4934d7e50c7824c6a05f59bbbe92c56c..4c1f8837c0c6c70e2f3b84ec68292d8309851b6e 100644
--- a/native/src/dictionary.h
+++ b/native/src/dictionary.h
@@ -17,6 +17,9 @@
#ifndef LATINIME_DICTIONARY_H
#define LATINIME_DICTIONARY_H
+#include "bigram_dictionary.h"
+#include "unigram_dictionary.h"
+
namespace latinime {
// 22-bit address = ~4MB dictionary size limit, which on average would be about 200k-300k words
@@ -35,68 +38,32 @@ namespace latinime {
class Dictionary {
public:
- Dictionary(void *dict, int typedLetterMultipler, int fullWordMultiplier);
+ Dictionary(void *dict, int typedLetterMultipler, int fullWordMultiplier, int maxWordLength,
+ int maxWords, int maxAlternatives);
int getSuggestions(int *codes, int codesSize, unsigned short *outWords, int *frequencies,
- int maxWordLength, int maxWords, int maxAlternatives, int skipPos,
- int *nextLetters, int nextLettersSize);
+ int *nextLetters, int nextLettersSize) {
+ return mUnigramDictionary->getSuggestions(codes, codesSize, outWords, frequencies,
+ nextLetters, nextLettersSize);
+ }
+
+ // TODO: Call mBigramDictionary instead of mUnigramDictionary
int getBigrams(unsigned short *word, int length, int *codes, int codesSize,
unsigned short *outWords, int *frequencies, int maxWordLength, int maxBigrams,
- int maxAlternatives);
- bool isValidWord(unsigned short *word, int length);
+ int maxAlternatives) {
+ return mUnigramDictionary->getBigrams(word, length, codes, codesSize, outWords, frequencies,
+ maxWordLength, maxBigrams, maxAlternatives);
+ }
+ bool isValidWord(unsigned short *word, int length) {
+ return mUnigramDictionary->isValidWord(word, length);
+ }
void setAsset(void *asset) { mAsset = asset; }
void *getAsset() { return mAsset; }
~Dictionary();
private:
- void initSuggestions(int *codes, int codesSize, unsigned short *outWords, int *frequencies,
- int maxWordLength, int maxWords, int maxAlternatives);
- int getSuggestionCandidates(int inputLength, int maxWords, int skipPos, int *nextLetters,
- int nextLettersSize);
- void getVersionNumber();
- bool checkIfDictVersionIsLatest();
- int getAddress(int *pos);
- int getBigramAddress(int *pos, bool advance);
- int getFreq(int *pos);
- int getBigramFreq(int *pos);
- void searchForTerminalNode(int address, int frequency);
-
- bool getFirstBitOfByte(int *pos) { return (mDict[*pos] & 0x80) > 0; }
- bool getSecondBitOfByte(int *pos) { return (mDict[*pos] & 0x40) > 0; }
- bool getTerminal(int *pos) { return (mDict[*pos] & FLAG_TERMINAL_MASK) > 0; }
- int getCount(int *pos) { return mDict[(*pos)++] & 0xFF; }
- unsigned short getChar(int *pos);
- int wideStrLen(unsigned short *str);
-
- bool sameAsTyped(unsigned short *word, int length);
- bool checkFirstCharacter(unsigned short *word);
- bool addWord(unsigned short *word, int length, int frequency);
- bool addWordBigram(unsigned short *word, int length, int frequency);
- unsigned short toLowerCase(unsigned short c);
- void getWordsRec(int pos, int depth, int maxDepth, bool completion, int frequency,
- int inputIndex, int diffs, int skipPos, int *nextLetters, int nextLettersSize);
- int isValidWordRec(int pos, unsigned short *word, int offset, int length);
- void registerNextLetter(unsigned short c, int *nextLetters, int nextLettersSize);
-
- unsigned char *mDict;
void *mAsset;
-
- int *mFrequencies;
- int *mBigramFreq;
- int mMaxWords;
- int mMaxBigrams;
- int mMaxWordLength;
- unsigned short *mOutputChars;
- unsigned short *mBigramChars;
- int *mInputCodes;
- int mInputLength;
- int mMaxAlternatives;
- unsigned short mWord[128];
- int mMaxEditDistance;
-
- int mFullWordMultiplier;
- int mTypedLetterMultiplier;
- int mVersion;
- int mBigram;
+ BigramDictionary *mBigramDictionary;
+ UnigramDictionary *mUnigramDictionary;
};
// ----------------------------------------------------------------------------
diff --git a/native/src/unigram_dictionary.cpp b/native/src/unigram_dictionary.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..88382a94d7d21a3c5077141c6eafe3a067a989cb
--- /dev/null
+++ b/native/src/unigram_dictionary.cpp
@@ -0,0 +1,631 @@
+/*
+**
+** Copyright 2010, The Android Open Source Project
+**
+** Licensed under the Apache License, Version 2.0 (the "License");
+** you may not use this file except in compliance with the License.
+** You may obtain a copy of the License at
+**
+** http://www.apache.org/licenses/LICENSE-2.0
+**
+** Unless required by applicable law or agreed to in writing, software
+** distributed under the License is distributed on an "AS IS" BASIS,
+** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+** See the License for the specific language governing permissions and
+** limitations under the License.
+*/
+
+#include <stdio.h>
+#include <fcntl.h>
+#include <sys/mman.h>
+#include <string.h>
+
+#ifdef FLAG_DBG
+#define LOG_TAG "LatinIME: dictionary.cpp"
+#include <cutils/log.h>
+#define DEBUG_DICT 1
+#else // FLAG_DBG
+#define LOGI
+#define DEBUG_DICT 0
+#endif // FLAG_DBG
+
+#include "unigram_dictionary.h"
+#include "basechars.h"
+#include "char_utils.h"
+
+#define DICTIONARY_VERSION_MIN 200
+#define DICTIONARY_HEADER_SIZE 2
+#define NOT_VALID_WORD -99
+
+#define SUGGEST_MISSING_CHARACTERS true
+#define SUGGEST_MISSING_CHARACTERS_THRESHOLD 5
+
+
+namespace latinime {
+
+UnigramDictionary::UnigramDictionary(void *dict, int typedLetterMultiplier, int fullWordMultiplier,
+ int maxWordLength, int maxWords, int maxAlternatives, Dictionary *parentDictionary)
+ : MAX_WORD_LENGTH(maxWordLength),MAX_WORDS(maxWords), MAX_ALTERNATIVES(maxAlternatives)
+{
+ LOGI("UnigramDictionary - constructor");
+ mDict = (unsigned char*) dict;
+ mTypedLetterMultiplier = typedLetterMultiplier;
+ mFullWordMultiplier = fullWordMultiplier;
+ mParentDictionary = parentDictionary;
+ getVersionNumber();
+}
+
+UnigramDictionary::~UnigramDictionary()
+{
+}
+
+int UnigramDictionary::getSuggestions(int *codes, int codesSize, unsigned short *outWords, int *frequencies,
+ int *nextLetters, int nextLettersSize)
+{
+
+ initSuggestions(codes, codesSize, outWords, frequencies);
+
+ int suggestedWordsCount = getSuggestionCandidates(codesSize, -1, nextLetters,
+ nextLettersSize);
+
+ // If there aren't sufficient suggestions, search for words by allowing wild cards at
+ // the different character positions. This feature is not ready for prime-time as we need
+ // to figure out the best ranking for such words compared to proximity corrections and
+ // completions.
+ if (SUGGEST_MISSING_CHARACTERS && suggestedWordsCount < SUGGEST_MISSING_CHARACTERS_THRESHOLD) {
+ for (int i = 0; i < codesSize; ++i) {
+ int tempCount = getSuggestionCandidates(codesSize, i, NULL, 0);
+ if (tempCount > suggestedWordsCount) {
+ suggestedWordsCount = tempCount;
+ break;
+ }
+ }
+ }
+
+ if (DEBUG_DICT) {
+ LOGI("Returning %d words", suggestedWordsCount);
+ LOGI("Next letters: ");
+ for (int k = 0; k < nextLettersSize; k++) {
+ if (nextLetters[k] > 0) {
+ LOGI("%c = %d,", k, nextLetters[k]);
+ }
+ }
+ LOGI("\n");
+ }
+ return suggestedWordsCount;
+}
+
+void UnigramDictionary::initSuggestions(int *codes, int codesSize, unsigned short *outWords,
+ int *frequencies) {
+ mFrequencies = frequencies;
+ mOutputChars = outWords;
+ mInputCodes = codes;
+ mInputLength = codesSize;
+ mMaxEditDistance = mInputLength < 5 ? 2 : mInputLength / 2;
+}
+
+int UnigramDictionary::getSuggestionCandidates(int inputLength, int skipPos,
+ int *nextLetters, int nextLettersSize) {
+ if (checkIfDictVersionIsLatest()) {
+ getWordsRec(DICTIONARY_HEADER_SIZE, 0, inputLength * 3, false, 1, 0, 0, skipPos,
+ nextLetters, nextLettersSize);
+ } else {
+ getWordsRec(0, 0, inputLength * 3, false, 1, 0, 0, skipPos, nextLetters, nextLettersSize);
+ }
+
+ // Get the word count
+ int suggestedWordsCount = 0;
+ while (suggestedWordsCount < MAX_WORDS && mFrequencies[suggestedWordsCount] > 0) {
+ suggestedWordsCount++;
+ }
+ return suggestedWordsCount;
+}
+
+void UnigramDictionary::registerNextLetter(unsigned short c, int *nextLetters, int nextLettersSize) {
+ if (c < nextLettersSize) {
+ nextLetters[c]++;
+ }
+}
+
+// TODO: Should be const static variable calculate in the constructor
+void
+UnigramDictionary::getVersionNumber()
+{
+ mVersion = (mDict[0] & 0xFF);
+ mBigram = (mDict[1] & 0xFF);
+ LOGI("IN NATIVE SUGGEST Version: %d Bigram : %d \n", mVersion, mBigram);
+}
+
+// TODO: Should be const static variable calculate in the constructor
+// Checks whether it has the latest dictionary or the old dictionary
+bool
+UnigramDictionary::checkIfDictVersionIsLatest()
+{
+ return (mVersion >= DICTIONARY_VERSION_MIN) && (mBigram == 1 || mBigram == 0);
+}
+
+unsigned short
+UnigramDictionary::getChar(int *pos)
+{
+ unsigned short ch = (unsigned short) (mDict[(*pos)++] & 0xFF);
+ // If the code is 255, then actual 16 bit code follows (in big endian)
+ if (ch == 0xFF) {
+ ch = ((mDict[*pos] & 0xFF) << 8) | (mDict[*pos + 1] & 0xFF);
+ (*pos) += 2;
+ }
+ return ch;
+}
+
+int
+UnigramDictionary::getAddress(int *pos)
+{
+ int address = 0;
+ if ((mDict[*pos] & FLAG_ADDRESS_MASK) == 0) {
+ *pos += 1;
+ } else {
+ address += (mDict[*pos] & (ADDRESS_MASK >> 16)) << 16;
+ address += (mDict[*pos + 1] & 0xFF) << 8;
+ address += (mDict[*pos + 2] & 0xFF);
+ *pos += 3;
+ }
+ return address;
+}
+
+int
+UnigramDictionary::getFreq(int *pos)
+{
+ int freq = mDict[(*pos)++] & 0xFF;
+
+ if (checkIfDictVersionIsLatest()) {
+ // skipping bigram
+ int bigramExist = (mDict[*pos] & FLAG_BIGRAM_READ);
+ if (bigramExist > 0) {
+ int nextBigramExist = 1;
+ while (nextBigramExist > 0) {
+ (*pos) += 3;
+ nextBigramExist = (mDict[(*pos)++] & FLAG_BIGRAM_CONTINUED);
+ }
+ } else {
+ (*pos)++;
+ }
+ }
+
+ return freq;
+}
+
+int
+UnigramDictionary::wideStrLen(unsigned short *str)
+{
+ if (!str) return 0;
+ unsigned short *end = str;
+ while (*end)
+ end++;
+ return end - str;
+}
+
+bool
+UnigramDictionary::addWord(unsigned short *word, int length, int frequency)
+{
+ word[length] = 0;
+ if (DEBUG_DICT) {
+ char s[length + 1];
+ for (int i = 0; i <= length; i++) s[i] = word[i];
+ LOGI("Found word = %s, freq = %d : \n", s, frequency);
+ }
+
+ // Find the right insertion point
+ int insertAt = 0;
+ while (insertAt < MAX_WORDS) {
+ if (frequency > mFrequencies[insertAt]
+ || (mFrequencies[insertAt] == frequency
+ && length < wideStrLen(mOutputChars + insertAt * MAX_WORD_LENGTH))) {
+ break;
+ }
+ insertAt++;
+ }
+ if (insertAt < MAX_WORDS) {
+ memmove((char*) mFrequencies + (insertAt + 1) * sizeof(mFrequencies[0]),
+ (char*) mFrequencies + insertAt * sizeof(mFrequencies[0]),
+ (MAX_WORDS - insertAt - 1) * sizeof(mFrequencies[0]));
+ mFrequencies[insertAt] = frequency;
+ memmove((char*) mOutputChars + (insertAt + 1) * MAX_WORD_LENGTH * sizeof(short),
+ (char*) mOutputChars + (insertAt ) * MAX_WORD_LENGTH * sizeof(short),
+ (MAX_WORDS - insertAt - 1) * sizeof(short) * MAX_WORD_LENGTH);
+ unsigned short *dest = mOutputChars + (insertAt ) * MAX_WORD_LENGTH;
+ while (length--) {
+ *dest++ = *word++;
+ }
+ *dest = 0; // NULL terminate
+ if (DEBUG_DICT) LOGI("Added word at %d\n", insertAt);
+ return true;
+ }
+ return false;
+}
+
+bool
+UnigramDictionary::addWordBigram(unsigned short *word, int length, int frequency)
+{
+ word[length] = 0;
+ if (DEBUG_DICT) {
+ char s[length + 1];
+ for (int i = 0; i <= length; i++) s[i] = word[i];
+ LOGI("Bigram: Found word = %s, freq = %d : \n", s, frequency);
+ }
+
+ // Find the right insertion point
+ int insertAt = 0;
+ while (insertAt < mMaxBigrams) {
+ if (frequency > mBigramFreq[insertAt]
+ || (mBigramFreq[insertAt] == frequency
+ && length < wideStrLen(mBigramChars + insertAt * MAX_WORD_LENGTH))) {
+ break;
+ }
+ insertAt++;
+ }
+ LOGI("Bigram: InsertAt -> %d maxBigrams: %d\n", insertAt, mMaxBigrams);
+ if (insertAt < mMaxBigrams) {
+ memmove((char*) mBigramFreq + (insertAt + 1) * sizeof(mBigramFreq[0]),
+ (char*) mBigramFreq + insertAt * sizeof(mBigramFreq[0]),
+ (mMaxBigrams - insertAt - 1) * sizeof(mBigramFreq[0]));
+ mBigramFreq[insertAt] = frequency;
+ memmove((char*) mBigramChars + (insertAt + 1) * MAX_WORD_LENGTH * sizeof(short),
+ (char*) mBigramChars + (insertAt ) * MAX_WORD_LENGTH * sizeof(short),
+ (mMaxBigrams - insertAt - 1) * sizeof(short) * MAX_WORD_LENGTH);
+ unsigned short *dest = mBigramChars + (insertAt ) * MAX_WORD_LENGTH;
+ while (length--) {
+ *dest++ = *word++;
+ }
+ *dest = 0; // NULL terminate
+ if (DEBUG_DICT) LOGI("Bigram: Added word at %d\n", insertAt);
+ return true;
+ }
+ return false;
+}
+
+unsigned short
+UnigramDictionary::toLowerCase(unsigned short c) {
+ if (c < sizeof(BASE_CHARS) / sizeof(BASE_CHARS[0])) {
+ c = BASE_CHARS[c];
+ }
+ if (c >='A' && c <= 'Z') {
+ c |= 32;
+ } else if (c > 127) {
+ c = latin_tolower(c);
+ }
+ return c;
+}
+
+bool
+UnigramDictionary::sameAsTyped(unsigned short *word, int length)
+{
+ if (length != mInputLength) {
+ return false;
+ }
+ int *inputCodes = mInputCodes;
+ while (length--) {
+ if ((unsigned int) *inputCodes != (unsigned int) *word) {
+ return false;
+ }
+ inputCodes += MAX_ALTERNATIVES;
+ word++;
+ }
+ return true;
+}
+
+static char QUOTE = '\'';
+
+void
+UnigramDictionary::getWordsRec(int pos, int depth, int maxDepth, bool completion, int snr, int inputIndex,
+ int diffs, int skipPos, int *nextLetters, int nextLettersSize)
+{
+ // Optimization: Prune out words that are too long compared to how much was typed.
+ if (depth > maxDepth) {
+ return;
+ }
+ if (diffs > mMaxEditDistance) {
+ return;
+ }
+ int count = getCount(&pos);
+ int *currentChars = NULL;
+ if (mInputLength <= inputIndex) {
+ completion = true;
+ } else {
+ currentChars = mInputCodes + (inputIndex * MAX_ALTERNATIVES);
+ }
+
+ for (int i = 0; i < count; i++) {
+ // -- at char
+ unsigned short c = getChar(&pos);
+ // -- at flag/add
+ unsigned short lowerC = toLowerCase(c);
+ bool terminal = getTerminal(&pos);
+ int childrenAddress = getAddress(&pos);
+ // -- after address or flag
+ int freq = 1;
+ if (terminal) freq = getFreq(&pos);
+ // -- after add or freq
+
+ // If we are only doing completions, no need to look at the typed characters.
+ if (completion) {
+ mWord[depth] = c;
+ if (terminal) {
+ addWord(mWord, depth + 1, freq * snr);
+ if (depth >= mInputLength && skipPos < 0) {
+ registerNextLetter(mWord[mInputLength], nextLetters, nextLettersSize);
+ }
+ }
+ if (childrenAddress != 0) {
+ getWordsRec(childrenAddress, depth + 1, maxDepth, completion, snr, inputIndex,
+ diffs, skipPos, nextLetters, nextLettersSize);
+ }
+ } else if ((c == QUOTE && currentChars[0] != QUOTE) || skipPos == depth) {
+ // Skip the ' or other letter and continue deeper
+ mWord[depth] = c;
+ if (childrenAddress != 0) {
+ getWordsRec(childrenAddress, depth + 1, maxDepth, false, snr, inputIndex, diffs,
+ skipPos, nextLetters, nextLettersSize);
+ }
+ } else {
+ int j = 0;
+ while (currentChars[j] > 0) {
+ if (currentChars[j] == lowerC || currentChars[j] == c) {
+ int addedWeight = j == 0 ? mTypedLetterMultiplier : 1;
+ mWord[depth] = c;
+ if (mInputLength == inputIndex + 1) {
+ if (terminal) {
+ if (//INCLUDE_TYPED_WORD_IF_VALID ||
+ !sameAsTyped(mWord, depth + 1)) {
+ int finalFreq = freq * snr * addedWeight;
+ if (skipPos < 0) finalFreq *= mFullWordMultiplier;
+ addWord(mWord, depth + 1, finalFreq);
+ }
+ }
+ if (childrenAddress != 0) {
+ getWordsRec(childrenAddress, depth + 1,
+ maxDepth, true, snr * addedWeight, inputIndex + 1,
+ diffs + (j > 0), skipPos, nextLetters, nextLettersSize);
+ }
+ } else if (childrenAddress != 0) {
+ getWordsRec(childrenAddress, depth + 1, maxDepth,
+ false, snr * addedWeight, inputIndex + 1, diffs + (j > 0),
+ skipPos, nextLetters, nextLettersSize);
+ }
+ }
+ j++;
+ if (skipPos >= 0) break;
+ }
+ }
+ }
+}
+
+int
+UnigramDictionary::getBigramAddress(int *pos, bool advance)
+{
+ int address = 0;
+
+ address += (mDict[*pos] & 0x3F) << 16;
+ address += (mDict[*pos + 1] & 0xFF) << 8;
+ address += (mDict[*pos + 2] & 0xFF);
+
+ if (advance) {
+ *pos += 3;
+ }
+
+ return address;
+}
+
+int
+UnigramDictionary::getBigramFreq(int *pos)
+{
+ int freq = mDict[(*pos)++] & FLAG_BIGRAM_FREQ;
+
+ return freq;
+}
+
+
+int
+UnigramDictionary::getBigrams(unsigned short *prevWord, int prevWordLength, int *codes, int codesSize,
+ unsigned short *bigramChars, int *bigramFreq, int maxWordLength, int maxBigrams,
+ int maxAlternatives)
+{
+ mBigramFreq = bigramFreq;
+ mBigramChars = bigramChars;
+ mInputCodes = codes;
+ mInputLength = codesSize;
+ mMaxBigrams = maxBigrams;
+
+ if (mBigram == 1 && checkIfDictVersionIsLatest()) {
+ int pos = isValidWordRec(
+ DICTIONARY_HEADER_SIZE, prevWord, 0, prevWordLength);
+ LOGI("Pos -> %d\n", pos);
+ if (pos < 0) {
+ return 0;
+ }
+
+ int bigramCount = 0;
+ int bigramExist = (mDict[pos] & FLAG_BIGRAM_READ);
+ if (bigramExist > 0) {
+ int nextBigramExist = 1;
+ while (nextBigramExist > 0 && bigramCount < maxBigrams) {
+ int bigramAddress = getBigramAddress(&pos, true);
+ int frequency = (FLAG_BIGRAM_FREQ & mDict[pos]);
+ // search for all bigrams and store them
+ searchForTerminalNode(bigramAddress, frequency);
+ nextBigramExist = (mDict[pos++] & FLAG_BIGRAM_CONTINUED);
+ bigramCount++;
+ }
+ }
+
+ return bigramCount;
+ }
+ return 0;
+}
+
+void
+UnigramDictionary::searchForTerminalNode(int addressLookingFor, int frequency)
+{
+ // track word with such address and store it in an array
+ unsigned short word[MAX_WORD_LENGTH];
+
+ int pos;
+ int followDownBranchAddress = DICTIONARY_HEADER_SIZE;
+ bool found = false;
+ char followingChar = ' ';
+ int depth = -1;
+
+ while(!found) {
+ bool followDownAddressSearchStop = false;
+ bool firstAddress = true;
+ bool haveToSearchAll = true;
+
+ if (depth >= 0) {
+ word[depth] = (unsigned short) followingChar;
+ }
+ pos = followDownBranchAddress; // pos start at count
+ int count = mDict[pos] & 0xFF;
+ LOGI("count - %d\n",count);
+ pos++;
+ for (int i = 0; i < count; i++) {
+ // pos at data
+ pos++;
+ // pos now at flag
+ if (!getFirstBitOfByte(&pos)) { // non-terminal
+ if (!followDownAddressSearchStop) {
+ int addr = getBigramAddress(&pos, false);
+ if (addr > addressLookingFor) {
+ followDownAddressSearchStop = true;
+ if (firstAddress) {
+ firstAddress = false;
+ haveToSearchAll = true;
+ } else if (!haveToSearchAll) {
+ break;
+ }
+ } else {
+ followDownBranchAddress = addr;
+ followingChar = (char)(0xFF & mDict[pos-1]);
+ if (firstAddress) {
+ firstAddress = false;
+ haveToSearchAll = false;
+ }
+ }
+ }
+ pos += 3;
+ } else if (getFirstBitOfByte(&pos)) { // terminal
+ if (addressLookingFor == (pos-1)) { // found !!
+ depth++;
+ word[depth] = (0xFF & mDict[pos-1]);
+ found = true;
+ break;
+ }
+ if (getSecondBitOfByte(&pos)) { // address + freq (4 byte)
+ if (!followDownAddressSearchStop) {
+ int addr = getBigramAddress(&pos, false);
+ if (addr > addressLookingFor) {
+ followDownAddressSearchStop = true;
+ if (firstAddress) {
+ firstAddress = false;
+ haveToSearchAll = true;
+ } else if (!haveToSearchAll) {
+ break;
+ }
+ } else {
+ followDownBranchAddress = addr;
+ followingChar = (char)(0xFF & mDict[pos-1]);
+ if (firstAddress) {
+ firstAddress = false;
+ haveToSearchAll = true;
+ }
+ }
+ }
+ pos += 4;
+ } else { // freq only (2 byte)
+ pos += 2;
+ }
+
+ // skipping bigram
+ int bigramExist = (mDict[pos] & FLAG_BIGRAM_READ);
+ if (bigramExist > 0) {
+ int nextBigramExist = 1;
+ while (nextBigramExist > 0) {
+ pos += 3;
+ nextBigramExist = (mDict[pos++] & FLAG_BIGRAM_CONTINUED);
+ }
+ } else {
+ pos++;
+ }
+ }
+ }
+ depth++;
+ if (followDownBranchAddress == 0) {
+ LOGI("ERROR!!! Cannot find bigram!!");
+ break;
+ }
+ }
+ if (checkFirstCharacter(word)) {
+ addWordBigram(word, depth, frequency);
+ }
+}
+
+bool
+UnigramDictionary::checkFirstCharacter(unsigned short *word)
+{
+ // Checks whether this word starts with same character or neighboring characters of
+ // what user typed.
+
+ int *inputCodes = mInputCodes;
+ int maxAlt = MAX_ALTERNATIVES;
+ while (maxAlt > 0) {
+ if ((unsigned int) *inputCodes == (unsigned int) *word) {
+ return true;
+ }
+ inputCodes++;
+ maxAlt--;
+ }
+ return false;
+}
+
+// TODO: Move to parent dictionary
+bool
+UnigramDictionary::isValidWord(unsigned short *word, int length)
+{
+ if (checkIfDictVersionIsLatest()) {
+ return (isValidWordRec(DICTIONARY_HEADER_SIZE, word, 0, length) != NOT_VALID_WORD);
+ } else {
+ return (isValidWordRec(0, word, 0, length) != NOT_VALID_WORD);
+ }
+}
+
+int
+UnigramDictionary::isValidWordRec(int pos, unsigned short *word, int offset, int length) {
+ // returns address of bigram data of that word
+ // return -99 if not found
+
+ int count = getCount(&pos);
+ unsigned short currentChar = (unsigned short) word[offset];
+ for (int j = 0; j < count; j++) {
+ unsigned short c = getChar(&pos);
+ int terminal = getTerminal(&pos);
+ int childPos = getAddress(&pos);
+ if (c == currentChar) {
+ if (offset == length - 1) {
+ if (terminal) {
+ return (pos+1);
+ }
+ } else {
+ if (childPos != 0) {
+ int t = isValidWordRec(childPos, word, offset + 1, length);
+ if (t > 0) {
+ return t;
+ }
+ }
+ }
+ }
+ if (terminal) {
+ getFreq(&pos);
+ }
+ // There could be two instances of each alphabet - upper and lower case. So continue
+ // looking ...
+ }
+ return NOT_VALID_WORD;
+}
+} // namespace latinime
diff --git a/native/src/unigram_dictionary.h b/native/src/unigram_dictionary.h
new file mode 100644
index 0000000000000000000000000000000000000000..28e4308b49444ca43b931403a51393c683c105c8
--- /dev/null
+++ b/native/src/unigram_dictionary.h
@@ -0,0 +1,104 @@
+/*
+ * Copyright (C) 2010 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LATINIME_UNIGRAM_DICTIONARY_H
+#define LATINIME_UNIGRAM_DICTIONARY_H
+
+namespace latinime {
+
+// 22-bit address = ~4MB dictionary size limit, which on average would be about 200k-300k words
+#define ADDRESS_MASK 0x3FFFFF
+
+// The bit that decides if an address follows in the next 22 bits
+#define FLAG_ADDRESS_MASK 0x40
+// The bit that decides if this is a terminal node for a word. The node could still have children,
+// if the word has other endings.
+#define FLAG_TERMINAL_MASK 0x80
+
+#define FLAG_BIGRAM_READ 0x80
+#define FLAG_BIGRAM_CHILDEXIST 0x40
+#define FLAG_BIGRAM_CONTINUED 0x80
+#define FLAG_BIGRAM_FREQ 0x7F
+
+class Dictionary;
+class UnigramDictionary {
+public:
+ UnigramDictionary(void *dict, int typedLetterMultipler, int fullWordMultiplier, int maxWordLength,
+ int maxWords, int maxAlternatives, Dictionary *parentDictionary);
+ int getSuggestions(int *codes, int codesSize, unsigned short *outWords, int *frequencies,
+ int *nextLetters, int nextLettersSize);
+ int getBigrams(unsigned short *word, int length, int *codes, int codesSize,
+ unsigned short *outWords, int *frequencies, int maxWordLength, int maxBigrams,
+ int maxAlternatives);
+ bool isValidWord(unsigned short *word, int length);
+ ~UnigramDictionary();
+
+private:
+ void initSuggestions(int *codes, int codesSize, unsigned short *outWords, int *frequencies);
+ int getSuggestionCandidates(int inputLength, int skipPos, int *nextLetters, int nextLettersSize);
+ void getVersionNumber();
+ bool checkIfDictVersionIsLatest();
+ int getAddress(int *pos);
+ int getBigramAddress(int *pos, bool advance);
+ int getFreq(int *pos);
+ int getBigramFreq(int *pos);
+ void searchForTerminalNode(int address, int frequency);
+
+ bool getFirstBitOfByte(int *pos) { return (mDict[*pos] & 0x80) > 0; }
+ bool getSecondBitOfByte(int *pos) { return (mDict[*pos] & 0x40) > 0; }
+ bool getTerminal(int *pos) { return (mDict[*pos] & FLAG_TERMINAL_MASK) > 0; }
+ int getCount(int *pos) { return mDict[(*pos)++] & 0xFF; }
+ unsigned short getChar(int *pos);
+ int wideStrLen(unsigned short *str);
+
+ bool sameAsTyped(unsigned short *word, int length);
+ bool checkFirstCharacter(unsigned short *word);
+ bool addWord(unsigned short *word, int length, int frequency);
+ bool addWordBigram(unsigned short *word, int length, int frequency);
+ unsigned short toLowerCase(unsigned short c);
+ void getWordsRec(int pos, int depth, int maxDepth, bool completion, int frequency,
+ int inputIndex, int diffs, int skipPos, int *nextLetters, int nextLettersSize);
+ void registerNextLetter(unsigned short c, int *nextLetters, int nextLettersSize);
+ int isValidWordRec(int pos, unsigned short *word, int offset, int length);
+
+ unsigned char *mDict;
+ Dictionary *mParentDictionary;
+
+ const int MAX_WORDS;
+ const int MAX_WORD_LENGTH;
+ const int MAX_ALTERNATIVES;
+
+ int *mFrequencies;
+ int *mBigramFreq;
+ int mMaxBigrams;
+ unsigned short *mOutputChars;
+ unsigned short *mBigramChars;
+ int *mInputCodes;
+ int mInputLength;
+ unsigned short mWord[128];
+ int mMaxEditDistance;
+
+ int mFullWordMultiplier;
+ int mTypedLetterMultiplier;
+ int mVersion;
+ int mBigram;
+};
+
+// ----------------------------------------------------------------------------
+
+}; // namespace latinime
+
+#endif // LATINIME_UNIGRAM_DICTIONARY_H