Generate dict code for version 401.

Bug: 13406708
Change-Id: I769ac84d54d997e7aefab0c9c16727455a132e0b

native/jni/CleanupNativeFileList.mk

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 LATIN_IME_CORE_SRC_FILES :=
+LATIN_IME_CORE_SRC_FILES_BACKWARD_V401 :=
 LATIN_IME_CORE_TEST_FILES :=
 LATIN_IME_JNI_SRC_FILES :=
 LATIN_IME_SRC_DIR :=

native/jni/NativeFileList.mk

@@ -98,6 +98,27 @@ LATIN_IME_CORE_SRC_FILES := \
         log_utils.cpp \
         time_keeper.cpp)
 
+LATIN_IME_CORE_SRC_FILES_BACKWARD_V401 := \
+    $(addprefix suggest/policyimpl/dictionary/structure/backward/v401/, \
+        ver4_dict_buffers.cpp \
+        ver4_dict_constants.cpp \
+        ver4_patricia_trie_node_reader.cpp \
+        ver4_patricia_trie_node_writer.cpp \
+        ver4_patricia_trie_policy.cpp \
+        ver4_patricia_trie_reading_utils.cpp \
+        ver4_patricia_trie_writing_helper.cpp \
+        ver4_pt_node_array_reader.cpp) \
+    $(addprefix suggest/policyimpl/dictionary/structure/backward/v401/content/, \
+        bigram_dict_content.cpp \
+        probability_dict_content.cpp \
+        shortcut_dict_content.cpp \
+        sparse_table_dict_content.cpp \
+        terminal_position_lookup_table.cpp) \
+    $(addprefix suggest/policyimpl/dictionary/structure/backward/v401/bigram/, \
+        ver4_bigram_list_policy.cpp)
+
+LATIN_IME_CORE_SRC_FILES += $(LATIN_IME_CORE_SRC_FILES_BACKWARD_V401)
+
 LATIN_IME_CORE_TEST_FILES := \
     defines_test.cpp \
     suggest/core/layout/normal_distribution_2d_test.cpp \

native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/Readme.txt

+Files under this directory have been auto generated.

native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/bigram/ver4_bigram_list_policy.cpp

+/*
+ * Copyright (C) 2013 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * !!!!! DO NOT CHANGE THE LOGIC IN THIS FILE !!!!!
+ * Do not edit this file other than updating policy's interface.
+ *
+ * This file was generated from
+ *   suggest/policyimpl/dictionary/structure/v4/bigram/ver4_bigram_list_policy.cpp
+ */
+
+#include "suggest/policyimpl/dictionary/structure/backward/v401/bigram/ver4_bigram_list_policy.h"
+
+#include "suggest/core/dictionary/property/bigram_property.h"
+#include "suggest/policyimpl/dictionary/header/header_policy.h"
+#include "suggest/policyimpl/dictionary/structure/pt_common/bigram/bigram_list_read_write_utils.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v401/content/bigram_dict_content.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v401/content/terminal_position_lookup_table.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_constants.h"
+#include "suggest/policyimpl/dictionary/utils/forgetting_curve_utils.h"
+
+namespace latinime {
+namespace backward {
+namespace v401 {
+
+void Ver4BigramListPolicy::getNextBigram(int *const outBigramPos, int *const outProbability,
+        bool *const outHasNext, int *const bigramEntryPos) const {
+    const BigramEntry bigramEntry =
+            mBigramDictContent->getBigramEntryAndAdvancePosition(bigramEntryPos);
+    if (outBigramPos) {
+        // Lookup target PtNode position.
+        *outBigramPos = mTerminalPositionLookupTable->getTerminalPtNodePosition(
+                bigramEntry.getTargetTerminalId());
+    }
+    if (outProbability) {
+        if (bigramEntry.hasHistoricalInfo()) {
+            *outProbability =
+                    ForgettingCurveUtils::decodeProbability(bigramEntry.getHistoricalInfo(),
+                            mHeaderPolicy);
+        } else {
+            *outProbability = bigramEntry.getProbability();
+        }
+    }
+    if (outHasNext) {
+        *outHasNext = bigramEntry.hasNext();
+    }
+}
+
+bool Ver4BigramListPolicy::addNewEntry(const int terminalId, const int newTargetTerminalId,
+        const BigramProperty *const bigramProperty, bool *const outAddedNewEntry) {
+    // 1. The word has no bigrams yet.
+    // 2. The word has bigrams, and there is the target in the list.
+    // 3. The word has bigrams, and there is an invalid entry that can be reclaimed.
+    // 4. The word has bigrams. We have to append new bigram entry to the list.
+    // 5. Same as 4, but the list is the last entry of the content file.
+    if (outAddedNewEntry) {
+        *outAddedNewEntry = false;
+    }
+    const int bigramListPos = mBigramDictContent->getBigramListHeadPos(terminalId);
+    if (bigramListPos == NOT_A_DICT_POS) {
+        // Case 1. PtNode that doesn't have a bigram list.
+        // Create new bigram list.
+        if (!mBigramDictContent->createNewBigramList(terminalId)) {
+            return false;
+        }
+        const BigramEntry newBigramEntry(false /* hasNext */, NOT_A_PROBABILITY,
+                newTargetTerminalId);
+        const BigramEntry bigramEntryToWrite = createUpdatedBigramEntryFrom(&newBigramEntry,
+                bigramProperty);
+        // Write an entry.
+        const int writingPos =  mBigramDictContent->getBigramListHeadPos(terminalId);
+        if (!mBigramDictContent->writeBigramEntry(&bigramEntryToWrite, writingPos)) {
+            return false;
+        }
+        if (outAddedNewEntry) {
+            *outAddedNewEntry = true;
+        }
+        return true;
+    }
+
+    int tailEntryPos = NOT_A_DICT_POS;
+    const int entryPosToUpdate = getEntryPosToUpdate(newTargetTerminalId, bigramListPos,
+            &tailEntryPos);
+    if (tailEntryPos != NOT_A_DICT_POS || entryPosToUpdate == NOT_A_DICT_POS) {
+        // Case 4, 5.
+        // Add new entry to the bigram list.
+        if (tailEntryPos == NOT_A_DICT_POS) {
+            // Case 4. Create new bigram list.
+            if (!mBigramDictContent->createNewBigramList(terminalId)) {
+                return false;
+            }
+            const int destPos = mBigramDictContent->getBigramListHeadPos(terminalId);
+            // Copy existing bigram list.
+            if (!mBigramDictContent->copyBigramList(bigramListPos, destPos, &tailEntryPos)) {
+                return false;
+            }
+        }
+        // Write new entry at the tail position of the bigram content.
+        const BigramEntry newBigramEntry(false /* hasNext */, NOT_A_PROBABILITY,
+                newTargetTerminalId);
+        const BigramEntry bigramEntryToWrite = createUpdatedBigramEntryFrom(
+                &newBigramEntry, bigramProperty);
+        if (!mBigramDictContent->writeBigramEntryAtTail(&bigramEntryToWrite)) {
+            return false;
+        }
+        // Update has next flag of the tail entry.
+        if (!updateHasNextFlag(true /* hasNext */, tailEntryPos)) {
+            return false;
+        }
+        if (outAddedNewEntry) {
+            *outAddedNewEntry = true;
+        }
+        return true;
+    }
+
+    // Case 2. Overwrite the existing entry. Case 3. Reclaim and reuse the existing invalid entry.
+    const BigramEntry originalBigramEntry = mBigramDictContent->getBigramEntry(entryPosToUpdate);
+    if (!originalBigramEntry.isValid()) {
+        // Case 3. Reuse the existing invalid entry. outAddedNewEntry is false when an existing
+        // entry is updated.
+        if (outAddedNewEntry) {
+            *outAddedNewEntry = true;
+        }
+    }
+    const BigramEntry updatedBigramEntry =
+            originalBigramEntry.updateTargetTerminalIdAndGetEntry(newTargetTerminalId);
+    const BigramEntry bigramEntryToWrite = createUpdatedBigramEntryFrom(
+            &updatedBigramEntry, bigramProperty);
+    return mBigramDictContent->writeBigramEntry(&bigramEntryToWrite, entryPosToUpdate);
+}
+
+bool Ver4BigramListPolicy::removeEntry(const int terminalId, const int targetTerminalId) {
+    const int bigramListPos = mBigramDictContent->getBigramListHeadPos(terminalId);
+    if (bigramListPos == NOT_A_DICT_POS) {
+        // Bigram list doesn't exist.
+        return false;
+    }
+    const int entryPosToUpdate = getEntryPosToUpdate(targetTerminalId, bigramListPos,
+            nullptr /* outTailEntryPos */);
+    if (entryPosToUpdate == NOT_A_DICT_POS) {
+        // Bigram entry doesn't exist.
+        return false;
+    }
+    const BigramEntry bigramEntry = mBigramDictContent->getBigramEntry(entryPosToUpdate);
+    if (targetTerminalId != bigramEntry.getTargetTerminalId()) {
+        // Bigram entry doesn't exist.
+        return false;
+    }
+    // Remove bigram entry by marking it as invalid entry and overwriting the original entry.
+    const BigramEntry updatedBigramEntry = bigramEntry.getInvalidatedEntry();
+    return mBigramDictContent->writeBigramEntry(&updatedBigramEntry, entryPosToUpdate);
+}
+
+bool Ver4BigramListPolicy::updateAllBigramEntriesAndDeleteUselessEntries(const int terminalId,
+        int *const outBigramCount) {
+    const int bigramListPos = mBigramDictContent->getBigramListHeadPos(terminalId);
+    if (bigramListPos == NOT_A_DICT_POS) {
+        // Bigram list doesn't exist.
+        return true;
+    }
+    bool hasNext = true;
+    int readingPos = bigramListPos;
+    while (hasNext) {
+        const int entryPos = readingPos;
+        const BigramEntry bigramEntry =
+                mBigramDictContent->getBigramEntryAndAdvancePosition(&readingPos);
+        hasNext = bigramEntry.hasNext();
+        if (!bigramEntry.isValid()) {
+            continue;
+        }
+        const int targetPtNodePos = mTerminalPositionLookupTable->getTerminalPtNodePosition(
+                bigramEntry.getTargetTerminalId());
+        if (targetPtNodePos == NOT_A_DICT_POS) {
+            // Invalidate bigram entry.
+            const BigramEntry updatedBigramEntry = bigramEntry.getInvalidatedEntry();
+            if (!mBigramDictContent->writeBigramEntry(&updatedBigramEntry, entryPos)) {
+                return false;
+            }
+        } else if (bigramEntry.hasHistoricalInfo()) {
+            const HistoricalInfo historicalInfo = ForgettingCurveUtils::createHistoricalInfoToSave(
+                    bigramEntry.getHistoricalInfo(), mHeaderPolicy);
+            if (ForgettingCurveUtils::needsToKeep(&historicalInfo, mHeaderPolicy)) {
+                const BigramEntry updatedBigramEntry =
+                        bigramEntry.updateHistoricalInfoAndGetEntry(&historicalInfo);
+                if (!mBigramDictContent->writeBigramEntry(&updatedBigramEntry, entryPos)) {
+                    return false;
+                }
+                *outBigramCount += 1;
+            } else {
+                // Remove entry.
+                const BigramEntry updatedBigramEntry = bigramEntry.getInvalidatedEntry();
+                if (!mBigramDictContent->writeBigramEntry(&updatedBigramEntry, entryPos)) {
+                    return false;
+                }
+            }
+        } else {
+            *outBigramCount += 1;
+        }
+    }
+    return true;
+}
+
+int Ver4BigramListPolicy::getBigramEntryConut(const int terminalId) {
+    const int bigramListPos = mBigramDictContent->getBigramListHeadPos(terminalId);
+    if (bigramListPos == NOT_A_DICT_POS) {
+        // Bigram list doesn't exist.
+        return 0;
+    }
+    int bigramCount = 0;
+    bool hasNext = true;
+    int readingPos = bigramListPos;
+    while (hasNext) {
+        const BigramEntry bigramEntry =
+                mBigramDictContent->getBigramEntryAndAdvancePosition(&readingPos);
+        hasNext = bigramEntry.hasNext();
+        if (bigramEntry.isValid()) {
+            bigramCount++;
+        }
+    }
+    return bigramCount;
+}
+
+int Ver4BigramListPolicy::getEntryPosToUpdate(const int targetTerminalIdToFind,
+        const int bigramListPos, int *const outTailEntryPos) const {
+    if (outTailEntryPos) {
+        *outTailEntryPos = NOT_A_DICT_POS;
+    }
+    bool hasNext = true;
+    int invalidEntryPos = NOT_A_DICT_POS;
+    int readingPos = bigramListPos;
+    while (hasNext) {
+        const int entryPos = readingPos;
+        const BigramEntry bigramEntry =
+                mBigramDictContent->getBigramEntryAndAdvancePosition(&readingPos);
+        hasNext = bigramEntry.hasNext();
+        if (bigramEntry.getTargetTerminalId() == targetTerminalIdToFind) {
+            // Entry with same target is found.
+            return entryPos;
+        } else if (!bigramEntry.isValid()) {
+            // Invalid entry that can be reused is found.
+            invalidEntryPos = entryPos;
+        }
+        if (!hasNext && mBigramDictContent->isContentTailPos(readingPos)) {
+            if (outTailEntryPos) {
+                *outTailEntryPos = entryPos;
+            }
+        }
+    }
+    return invalidEntryPos;
+}
+
+const BigramEntry Ver4BigramListPolicy::createUpdatedBigramEntryFrom(
+        const BigramEntry *const originalBigramEntry,
+        const BigramProperty *const bigramProperty) const {
+    // TODO: Consolidate historical info and probability.
+    if (mHeaderPolicy->hasHistoricalInfoOfWords()) {
+        const HistoricalInfo historicalInfoForUpdate(bigramProperty->getTimestamp(),
+                bigramProperty->getLevel(), bigramProperty->getCount());
+        const HistoricalInfo updatedHistoricalInfo =
+                ForgettingCurveUtils::createUpdatedHistoricalInfo(
+                        originalBigramEntry->getHistoricalInfo(), bigramProperty->getProbability(),
+                        &historicalInfoForUpdate, mHeaderPolicy);
+        return originalBigramEntry->updateHistoricalInfoAndGetEntry(&updatedHistoricalInfo);
+    } else {
+        return originalBigramEntry->updateProbabilityAndGetEntry(bigramProperty->getProbability());
+    }
+}
+
+bool Ver4BigramListPolicy::updateHasNextFlag(const bool hasNext, const int bigramEntryPos) {
+    const BigramEntry bigramEntry = mBigramDictContent->getBigramEntry(bigramEntryPos);
+    const BigramEntry updatedBigramEntry = bigramEntry.updateHasNextAndGetEntry(hasNext);
+    return mBigramDictContent->writeBigramEntry(&updatedBigramEntry, bigramEntryPos);
+}
+
+} // namespace v401
+} // namespace backward
+} // namespace latinime

native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/bigram/ver4_bigram_list_policy.h

+/*
+ * Copyright (C) 2013 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * !!!!! DO NOT CHANGE THE LOGIC IN THIS FILE !!!!!
+ * Do not edit this file other than updating policy's interface.
+ *
+ * This file was generated from
+ *   suggest/policyimpl/dictionary/structure/v4/bigram/ver4_bigram_list_policy.h
+ */
+
+#ifndef LATINIME_BACKWARD_V401_VER4_BIGRAM_LIST_POLICY_H
+#define LATINIME_BACKWARD_V401_VER4_BIGRAM_LIST_POLICY_H
+
+#include "defines.h"
+#include "suggest/core/policy/dictionary_bigrams_structure_policy.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v401/content/bigram_entry.h"
+
+namespace latinime {
+namespace backward {
+namespace v401 {
+
+class BigramDictContent;
+} // namespace v401
+} // namespace backward
+class BigramProperty;
+namespace backward {
+namespace v401 {
+} // namespace v401
+} // namespace backward
+class HeaderPolicy;
+namespace backward {
+namespace v401 {
+class TerminalPositionLookupTable;
+
+class Ver4BigramListPolicy : public DictionaryBigramsStructurePolicy {
+ public:
+    Ver4BigramListPolicy(BigramDictContent *const bigramDictContent,
+            const TerminalPositionLookupTable *const terminalPositionLookupTable,
+            const HeaderPolicy *const headerPolicy)
+            : mBigramDictContent(bigramDictContent),
+              mTerminalPositionLookupTable(terminalPositionLookupTable),
+              mHeaderPolicy(headerPolicy) {}
+
+    void getNextBigram(int *const outBigramPos, int *const outProbability,
+            bool *const outHasNext, int *const bigramEntryPos) const;
+
+    void skipAllBigrams(int *const pos) const {
+        // Do nothing because we don't need to skip bigram lists in ver4 dictionaries.
+    }
+
+    bool addNewEntry(const int terminalId, const int newTargetTerminalId,
+            const BigramProperty *const bigramProperty, bool *const outAddedNewEntry);
+
+    bool removeEntry(const int terminalId, const int targetTerminalId);
+
+    bool updateAllBigramEntriesAndDeleteUselessEntries(const int terminalId,
+            int *const outBigramCount);
+
+    int getBigramEntryConut(const int terminalId);
+
+ private:
+    DISALLOW_IMPLICIT_CONSTRUCTORS(Ver4BigramListPolicy);
+
+    int getEntryPosToUpdate(const int targetTerminalIdToFind, const int bigramListPos,
+            int *const outTailEntryPos) const;
+
+    const BigramEntry createUpdatedBigramEntryFrom(const BigramEntry *const originalBigramEntry,
+            const BigramProperty *const bigramProperty) const;
+
+    bool updateHasNextFlag(const bool hasNext, const int bigramEntryPos);
+
+    BigramDictContent *const mBigramDictContent;
+    const TerminalPositionLookupTable *const mTerminalPositionLookupTable;
+    const HeaderPolicy *const mHeaderPolicy;
+};
+} // namespace v401
+} // namespace backward
+} // namespace latinime
+#endif /* LATINIME_BACKWARD_V401_VER4_BIGRAM_LIST_POLICY_H */

native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/bigram_dict_content.cpp

+/*
+ * Copyright (C) 2013 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * !!!!! DO NOT EDIT THIS FILE !!!!!
+ *
+ * This file was generated from
+ *   suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.cpp
+ */
+
+#include "suggest/policyimpl/dictionary/structure/backward/v401/content/bigram_dict_content.h"
+
+#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
+
+namespace latinime {
+namespace backward {
+namespace v401 {
+
+const BigramEntry BigramDictContent::getBigramEntryAndAdvancePosition(
+        int *const bigramEntryPos) const {
+    const BufferWithExtendableBuffer *const bigramListBuffer = getContentBuffer();
+    if (*bigramEntryPos < 0 || *bigramEntryPos >=  bigramListBuffer->getTailPosition()) {
+        AKLOGE("Invalid bigram entry position. bigramEntryPos: %d, bufSize: %d",
+                *bigramEntryPos, bigramListBuffer->getTailPosition());
+        ASSERT(false);
+        return BigramEntry(false /* hasNext */, NOT_A_PROBABILITY,
+                Ver4DictConstants::NOT_A_TERMINAL_ID);
+    }
+    const int bigramFlags = bigramListBuffer->readUintAndAdvancePosition(
+            Ver4DictConstants::BIGRAM_FLAGS_FIELD_SIZE, bigramEntryPos);
+    const bool hasNext = (bigramFlags & Ver4DictConstants::BIGRAM_HAS_NEXT_MASK) != 0;
+    int probability = NOT_A_PROBABILITY;
+    int timestamp = NOT_A_TIMESTAMP;
+    int level = 0;
+    int count = 0;
+    if (mHasHistoricalInfo) {
+        probability = bigramListBuffer->readUintAndAdvancePosition(
+                Ver4DictConstants::PROBABILITY_SIZE, bigramEntryPos);
+        timestamp = bigramListBuffer->readUintAndAdvancePosition(
+                Ver4DictConstants::TIME_STAMP_FIELD_SIZE, bigramEntryPos);
+        level = bigramListBuffer->readUintAndAdvancePosition(
+                Ver4DictConstants::WORD_LEVEL_FIELD_SIZE, bigramEntryPos);
+        count = bigramListBuffer->readUintAndAdvancePosition(
+                Ver4DictConstants::WORD_COUNT_FIELD_SIZE, bigramEntryPos);
+    } else {
+        probability = bigramFlags & Ver4DictConstants::BIGRAM_PROBABILITY_MASK;
+    }
+    const int encodedTargetTerminalId = bigramListBuffer->readUintAndAdvancePosition(
+            Ver4DictConstants::BIGRAM_TARGET_TERMINAL_ID_FIELD_SIZE, bigramEntryPos);
+    const int targetTerminalId =
+            (encodedTargetTerminalId == Ver4DictConstants::INVALID_BIGRAM_TARGET_TERMINAL_ID) ?
+                    Ver4DictConstants::NOT_A_TERMINAL_ID : encodedTargetTerminalId;
+    if (mHasHistoricalInfo) {
+        const HistoricalInfo historicalInfo(timestamp, level, count);
+        return BigramEntry(hasNext, probability, &historicalInfo, targetTerminalId);
+    } else {
+        return BigramEntry(hasNext, probability, targetTerminalId);
+    }
+}
+
+bool BigramDictContent::writeBigramEntryAndAdvancePosition(
+        const BigramEntry *const bigramEntryToWrite, int *const entryWritingPos) {
+    BufferWithExtendableBuffer *const bigramListBuffer = getWritableContentBuffer();
+    const int bigramFlags = createAndGetBigramFlags(
+            mHasHistoricalInfo ? 0 : bigramEntryToWrite->getProbability(),
+            bigramEntryToWrite->hasNext());
+    if (!bigramListBuffer->writeUintAndAdvancePosition(bigramFlags,
+            Ver4DictConstants::BIGRAM_FLAGS_FIELD_SIZE, entryWritingPos)) {
+        AKLOGE("Cannot write bigram flags. pos: %d, flags: %x", *entryWritingPos, bigramFlags);
+        return false;
+    }
+    if (mHasHistoricalInfo) {
+        if (!bigramListBuffer->writeUintAndAdvancePosition(bigramEntryToWrite->getProbability(),
+                Ver4DictConstants::PROBABILITY_SIZE, entryWritingPos)) {
+            AKLOGE("Cannot write bigram probability. pos: %d, probability: %d", *entryWritingPos,
+                    bigramEntryToWrite->getProbability());
+            return false;
+        }
+        const HistoricalInfo *const historicalInfo = bigramEntryToWrite->getHistoricalInfo();
+        if (!bigramListBuffer->writeUintAndAdvancePosition(historicalInfo->getTimeStamp(),
+                Ver4DictConstants::TIME_STAMP_FIELD_SIZE, entryWritingPos)) {
+            AKLOGE("Cannot write bigram timestamps. pos: %d, timestamp: %d", *entryWritingPos,
+                    historicalInfo->getTimeStamp());
+            return false;
+        }
+        if (!bigramListBuffer->writeUintAndAdvancePosition(historicalInfo->getLevel(),
+                Ver4DictConstants::WORD_LEVEL_FIELD_SIZE, entryWritingPos)) {
+            AKLOGE("Cannot write bigram level. pos: %d, level: %d", *entryWritingPos,
+                    historicalInfo->getLevel());
+            return false;
+        }
+        if (!bigramListBuffer->writeUintAndAdvancePosition(historicalInfo->getCount(),
+                Ver4DictConstants::WORD_COUNT_FIELD_SIZE, entryWritingPos)) {
+            AKLOGE("Cannot write bigram count. pos: %d, count: %d", *entryWritingPos,
+                    historicalInfo->getCount());
+            return false;
+        }
+    }
+    const int targetTerminalIdToWrite =
+            (bigramEntryToWrite->getTargetTerminalId() == Ver4DictConstants::NOT_A_TERMINAL_ID) ?
+                    Ver4DictConstants::INVALID_BIGRAM_TARGET_TERMINAL_ID :
+                            bigramEntryToWrite->getTargetTerminalId();
+    if (!bigramListBuffer->writeUintAndAdvancePosition(targetTerminalIdToWrite,
+            Ver4DictConstants::BIGRAM_TARGET_TERMINAL_ID_FIELD_SIZE, entryWritingPos)) {
+        AKLOGE("Cannot write bigram target terminal id. pos: %d, target terminal id: %d",
+                *entryWritingPos, bigramEntryToWrite->getTargetTerminalId());
+        return false;
+    }
+    return true;
+}
+
+bool BigramDictContent::copyBigramList(const int bigramListPos, const int toPos,
+        int *const outTailEntryPos) {
+    int readingPos = bigramListPos;
+    int writingPos = toPos;
+    bool hasNext = true;
+    while (hasNext) {
+        const BigramEntry bigramEntry = getBigramEntryAndAdvancePosition(&readingPos);
+        hasNext = bigramEntry.hasNext();
+        if (!hasNext) {
+            *outTailEntryPos = writingPos;
+        }
+        if (!writeBigramEntryAndAdvancePosition(&bigramEntry, &writingPos)) {
+            AKLOGE("Cannot write bigram entry to copy. pos: %d", writingPos);
+            return false;
+        }
+    }
+    return true;
+}
+
+bool BigramDictContent::runGC(const TerminalPositionLookupTable::TerminalIdMap *const terminalIdMap,
+        const BigramDictContent *const originalBigramDictContent,
+        int *const outBigramEntryCount) {
+    for (TerminalPositionLookupTable::TerminalIdMap::const_iterator it = terminalIdMap->begin();
+            it != terminalIdMap->end(); ++it) {
+        const int originalBigramListPos =
+                originalBigramDictContent->getBigramListHeadPos(it->first);
+        if (originalBigramListPos == NOT_A_DICT_POS) {
+            // This terminal does not have a bigram list.
+            continue;
+        }
+        const int bigramListPos = getContentBuffer()->getTailPosition();
+        int bigramEntryCount = 0;
+        // Copy bigram list with GC from original content.
+        if (!runGCBigramList(originalBigramListPos, originalBigramDictContent, bigramListPos,
+                terminalIdMap, &bigramEntryCount)) {
+            AKLOGE("Cannot complete GC for the bigram list. original pos: %d, pos: %d",
+                    originalBigramListPos, bigramListPos);
+            return false;
+        }
+        if (bigramEntryCount == 0) {
+            // All bigram entries are useless. This terminal does not have a bigram list.
+            continue;
+        }
+        *outBigramEntryCount += bigramEntryCount;
+        // Set bigram list position to the lookup table.
+        if (!getUpdatableAddressLookupTable()->set(it->second, bigramListPos)) {
+            AKLOGE("Cannot set bigram list position. terminal id: %d, pos: %d",
+                    it->second, bigramListPos);
+            return false;
+        }
+    }
+    return true;
+}
+
+// Returns whether GC for the bigram list was succeeded or not.
+bool BigramDictContent::runGCBigramList(const int bigramListPos,
+        const BigramDictContent *const sourceBigramDictContent, const int toPos,
+        const TerminalPositionLookupTable::TerminalIdMap *const terminalIdMap,
+        int *const outEntrycount) {
+    bool hasNext = true;
+    int readingPos = bigramListPos;
+    int writingPos = toPos;
+    int lastEntryPos = NOT_A_DICT_POS;
+    while (hasNext) {
+        const BigramEntry originalBigramEntry =
+                sourceBigramDictContent->getBigramEntryAndAdvancePosition(&readingPos);
+        hasNext = originalBigramEntry.hasNext();
+        if (originalBigramEntry.getTargetTerminalId() == Ver4DictConstants::NOT_A_TERMINAL_ID) {
+            continue;
+        }
+        TerminalPositionLookupTable::TerminalIdMap::const_iterator it =
+                terminalIdMap->find(originalBigramEntry.getTargetTerminalId());
+        if (it == terminalIdMap->end()) {
+            // Target word has been removed.
+            continue;
+        }
+        lastEntryPos = hasNext ? writingPos : NOT_A_DICT_POS;
+        const BigramEntry updatedBigramEntry =
+                originalBigramEntry.updateTargetTerminalIdAndGetEntry(it->second);
+        if (!writeBigramEntryAndAdvancePosition(&updatedBigramEntry, &writingPos)) {
+            AKLOGE("Cannot write bigram entry to run GC. pos: %d", writingPos);
+            return false;
+        }
+        *outEntrycount += 1;
+    }
+    if (lastEntryPos != NOT_A_DICT_POS) {
+        // Update has next flag in the last written entry.
+        const BigramEntry bigramEntry = getBigramEntry(lastEntryPos).updateHasNextAndGetEntry(
+                false /* hasNext */);
+        if (!writeBigramEntry(&bigramEntry, lastEntryPos)) {
+            AKLOGE("Cannot write bigram entry to set hasNext flag after GC. pos: %d", writingPos);
+            return false;
+        }
+    }
+    return true;
+}
+
+} // namespace v401
+} // namespace backward
+} // namespace latinime

native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/bigram_dict_content.h

+/*
+ * Copyright (C) 2013, The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * !!!!! DO NOT EDIT THIS FILE !!!!!
+ *
+ * This file was generated from
+ *   suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.h
+ */
+
+#ifndef LATINIME_BACKWARD_V401_BIGRAM_DICT_CONTENT_H
+#define LATINIME_BACKWARD_V401_BIGRAM_DICT_CONTENT_H
+
+#include "defines.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v401/content/bigram_entry.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v401/content/sparse_table_dict_content.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v401/content/terminal_position_lookup_table.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_constants.h"
+
+namespace latinime {
+namespace backward {
+namespace v401 {
+
+class BigramDictContent : public SparseTableDictContent {
+ public:
+    BigramDictContent(const char *const dictPath, const bool hasHistoricalInfo,
+            const bool isUpdatable)
+            : SparseTableDictContent(dictPath,
+                      Ver4DictConstants::BIGRAM_LOOKUP_TABLE_FILE_EXTENSION,
+                      Ver4DictConstants::BIGRAM_CONTENT_TABLE_FILE_EXTENSION,
+                      Ver4DictConstants::BIGRAM_FILE_EXTENSION, isUpdatable,
+                      Ver4DictConstants::BIGRAM_ADDRESS_TABLE_BLOCK_SIZE,
+                      Ver4DictConstants::BIGRAM_ADDRESS_TABLE_DATA_SIZE),
+              mHasHistoricalInfo(hasHistoricalInfo) {}
+
+    BigramDictContent(const bool hasHistoricalInfo)
+            : SparseTableDictContent(Ver4DictConstants::BIGRAM_ADDRESS_TABLE_BLOCK_SIZE,
+                      Ver4DictConstants::BIGRAM_ADDRESS_TABLE_DATA_SIZE),
+              mHasHistoricalInfo(hasHistoricalInfo) {}
+
+    const BigramEntry getBigramEntry(const int bigramEntryPos) const {
+        int readingPos = bigramEntryPos;
+        return getBigramEntryAndAdvancePosition(&readingPos);
+    }
+
+    const BigramEntry getBigramEntryAndAdvancePosition(int *const bigramEntryPos) const;
+
+    // Returns head position of bigram list for a PtNode specified by terminalId.
+    int getBigramListHeadPos(const int terminalId) const {
+        const SparseTable *const addressLookupTable = getAddressLookupTable();
+        if (!addressLookupTable->contains(terminalId)) {
+            return NOT_A_DICT_POS;
+        }
+        return addressLookupTable->get(terminalId);
+    }
+
+    bool writeBigramEntryAtTail(const BigramEntry *const bigramEntryToWrite) {
+        int writingPos = getContentBuffer()->getTailPosition();
+        return writeBigramEntryAndAdvancePosition(bigramEntryToWrite, &writingPos);
+    }
+
+    bool writeBigramEntry(const BigramEntry *const bigramEntryToWrite, const int entryWritingPos) {
+        int writingPos = entryWritingPos;
+        return writeBigramEntryAndAdvancePosition(bigramEntryToWrite, &writingPos);
+    }
+
+    bool writeBigramEntryAndAdvancePosition(const BigramEntry *const bigramEntryToWrite,
+            int *const entryWritingPos);
+
+    bool createNewBigramList(const int terminalId) {
+        const int bigramListPos = getContentBuffer()->getTailPosition();
+        return getUpdatableAddressLookupTable()->set(terminalId, bigramListPos);
+    }
+
+    bool copyBigramList(const int bigramListPos, const int toPos, int *const outTailEntryPos);
+
+    bool flushToFile(const char *const dictPath) const {
+        return flush(dictPath, Ver4DictConstants::BIGRAM_LOOKUP_TABLE_FILE_EXTENSION,
+                Ver4DictConstants::BIGRAM_CONTENT_TABLE_FILE_EXTENSION,
+                Ver4DictConstants::BIGRAM_FILE_EXTENSION);
+    }
+
+    bool runGC(const TerminalPositionLookupTable::TerminalIdMap *const terminalIdMap,
+            const BigramDictContent *const originalBigramDictContent,
+            int *const outBigramEntryCount);
+
+    bool isContentTailPos(const int pos) const {
+        return pos == getContentBuffer()->getTailPosition();
+    }
+
+ private:
+    DISALLOW_COPY_AND_ASSIGN(BigramDictContent);
+
+    int createAndGetBigramFlags(const int probability, const bool hasNext) const {
+        return (probability & Ver4DictConstants::BIGRAM_PROBABILITY_MASK)
+                | (hasNext ? Ver4DictConstants::BIGRAM_HAS_NEXT_MASK : 0);
+    }
+
+    bool runGCBigramList(const int bigramListPos,
+            const BigramDictContent *const sourceBigramDictContent, const int toPos,
+            const TerminalPositionLookupTable::TerminalIdMap *const terminalIdMap,
+            int *const outEntryCount);
+
+    bool mHasHistoricalInfo;
+};
+} // namespace v401
+} // namespace backward
+} // namespace latinime
+#endif /* LATINIME_BACKWARD_V401_BIGRAM_DICT_CONTENT_H */

native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/bigram_entry.h

+/*
+ * Copyright (C) 2013, The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * !!!!! DO NOT EDIT THIS FILE !!!!!
+ *
+ * This file was generated from
+ *   suggest/policyimpl/dictionary/structure/v4/content/bigram_entry.h
+ */
+
+#ifndef LATINIME_BACKWARD_V401_BIGRAM_ENTRY_H
+#define LATINIME_BACKWARD_V401_BIGRAM_ENTRY_H
+
+#include "defines.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_constants.h"
+#include "suggest/policyimpl/dictionary/utils/historical_info.h"
+
+namespace latinime {
+namespace backward {
+namespace v401 {
+
+class BigramEntry {
+ public:
+    BigramEntry(const BigramEntry& bigramEntry)
+            : mHasNext(bigramEntry.mHasNext), mProbability(bigramEntry.mProbability),
+              mHistoricalInfo(), mTargetTerminalId(bigramEntry.mTargetTerminalId) {}
+
+    // Entry with historical information.
+    BigramEntry(const bool hasNext, const int probability, const int targetTerminalId)
+            : mHasNext(hasNext), mProbability(probability), mHistoricalInfo(),
+              mTargetTerminalId(targetTerminalId) {}
+
+    // Entry with historical information.
+    BigramEntry(const bool hasNext, const int probability,
+            const HistoricalInfo *const historicalInfo, const int targetTerminalId)
+            : mHasNext(hasNext), mProbability(probability), mHistoricalInfo(*historicalInfo),
+              mTargetTerminalId(targetTerminalId) {}
+
+    const BigramEntry getInvalidatedEntry() const {
+        return updateTargetTerminalIdAndGetEntry(Ver4DictConstants::NOT_A_TERMINAL_ID);
+    }
+
+    const BigramEntry updateHasNextAndGetEntry(const bool hasNext) const {
+        return BigramEntry(hasNext, mProbability, &mHistoricalInfo, mTargetTerminalId);
+    }
+
+    const BigramEntry updateTargetTerminalIdAndGetEntry(const int newTargetTerminalId) const {
+        return BigramEntry(mHasNext, mProbability, &mHistoricalInfo, newTargetTerminalId);
+    }
+
+    const BigramEntry updateProbabilityAndGetEntry(const int probability) const {
+        return BigramEntry(mHasNext, probability, &mHistoricalInfo, mTargetTerminalId);
+    }
+
+    const BigramEntry updateHistoricalInfoAndGetEntry(
+            const HistoricalInfo *const historicalInfo) const {
+        return BigramEntry(mHasNext, mProbability, historicalInfo, mTargetTerminalId);
+    }
+
+    bool isValid() const {
+        return mTargetTerminalId != Ver4DictConstants::NOT_A_TERMINAL_ID;
+    }
+
+    bool hasNext() const {
+        return mHasNext;
+    }
+
+    int getProbability() const {
+        return mProbability;
+    }
+
+    bool hasHistoricalInfo() const {
+        return mHistoricalInfo.isValid();
+    }
+
+    const HistoricalInfo *getHistoricalInfo() const {
+        return &mHistoricalInfo;
+    }
+
+    int getTargetTerminalId() const {
+        return mTargetTerminalId;
+    }
+
+ private:
+    // Copy constructor is public to use this class as a type of return value.
+    DISALLOW_DEFAULT_CONSTRUCTOR(BigramEntry);
+    DISALLOW_ASSIGNMENT_OPERATOR(BigramEntry);
+
+    const bool mHasNext;
+    const int mProbability;
+    const HistoricalInfo mHistoricalInfo;
+    const int mTargetTerminalId;
+};
+} // namespace v401
+} // namespace backward
+} // namespace latinime
+#endif /* LATINIME_BACKWARD_V401_BIGRAM_ENTRY_H */

native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/dict_content.h

+/*
+ * Copyright (C) 2013, The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * !!!!! DO NOT EDIT THIS FILE !!!!!
+ *
+ * This file was generated from
+ *   suggest/policyimpl/dictionary/structure/v4/content/dict_content.h
+ */
+
+#ifndef LATINIME_BACKWARD_V401_DICT_CONTENT_H
+#define LATINIME_BACKWARD_V401_DICT_CONTENT_H
+
+#include "defines.h"
+
+namespace latinime {
+namespace backward {
+namespace v401 {
+
+class DictContent {
+ public:
+    virtual ~DictContent() {}
+    virtual bool isValid() const = 0;
+
+ protected:
+    DictContent() {}
+
+ private:
+    DISALLOW_COPY_AND_ASSIGN(DictContent);
+};
+} // namespace v401
+} // namespace backward
+} // namespace latinime
+#endif /* LATINIME_BACKWARD_V401_DICT_CONTENT_H */

native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/probability_dict_content.cpp

+/*
+ * Copyright (C) 2013 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * !!!!! DO NOT EDIT THIS FILE !!!!!
+ *
+ * This file was generated from
+ *   suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.cpp
+ */
+
+#include "suggest/policyimpl/dictionary/structure/backward/v401/content/probability_dict_content.h"
+
+#include "suggest/policyimpl/dictionary/structure/backward/v401/content/probability_entry.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v401/content/terminal_position_lookup_table.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_constants.h"
+#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
+
+namespace latinime {
+namespace backward {
+namespace v401 {
+
+const ProbabilityEntry ProbabilityDictContent::getProbabilityEntry(const int terminalId) const {
+    if (terminalId < 0 || terminalId >= mSize) {
+        // This method can be called with invalid terminal id during GC.
+        return ProbabilityEntry(0 /* flags */, NOT_A_PROBABILITY);
+    }
+    const BufferWithExtendableBuffer *const buffer = getBuffer();
+    int entryPos = getEntryPos(terminalId);
+    const int flags = buffer->readUintAndAdvancePosition(
+            Ver4DictConstants::FLAGS_IN_PROBABILITY_FILE_SIZE, &entryPos);
+    const int probability = buffer->readUintAndAdvancePosition(
+            Ver4DictConstants::PROBABILITY_SIZE, &entryPos);
+    if (mHasHistoricalInfo) {
+        const int timestamp = buffer->readUintAndAdvancePosition(
+                Ver4DictConstants::TIME_STAMP_FIELD_SIZE, &entryPos);
+        const int level = buffer->readUintAndAdvancePosition(
+                Ver4DictConstants::WORD_LEVEL_FIELD_SIZE, &entryPos);
+        const int count = buffer->readUintAndAdvancePosition(
+                Ver4DictConstants::WORD_COUNT_FIELD_SIZE, &entryPos);
+        const HistoricalInfo historicalInfo(timestamp, level, count);
+        return ProbabilityEntry(flags, probability, &historicalInfo);
+    } else {
+        return ProbabilityEntry(flags, probability);
+    }
+}
+
+bool ProbabilityDictContent::setProbabilityEntry(const int terminalId,
+        const ProbabilityEntry *const probabilityEntry) {
+    if (terminalId < 0) {
+        return false;
+    }
+    const int entryPos = getEntryPos(terminalId);
+    if (terminalId >= mSize) {
+        ProbabilityEntry dummyEntry;
+        // Write new entry.
+        int writingPos = getBuffer()->getTailPosition();
+        while (writingPos <= entryPos) {
+            // Fulfilling with dummy entries until writingPos.
+            if (!writeEntry(&dummyEntry, writingPos)) {
+                AKLOGE("Cannot write dummy entry. pos: %d, mSize: %d", writingPos, mSize);
+                return false;
+            }
+            writingPos += getEntrySize();
+            mSize++;
+        }
+    }
+    return writeEntry(probabilityEntry, entryPos);
+}
+
+bool ProbabilityDictContent::flushToFile(const char *const dictPath) const {
+    if (getEntryPos(mSize) < getBuffer()->getTailPosition()) {
+        ProbabilityDictContent probabilityDictContentToWrite(mHasHistoricalInfo);
+        for (int i = 0; i < mSize; ++i) {
+            const ProbabilityEntry probabilityEntry = getProbabilityEntry(i);
+            if (!probabilityDictContentToWrite.setProbabilityEntry(i, &probabilityEntry)) {
+                AKLOGE("Cannot set probability entry in flushToFile. terminalId: %d", i);
+                return false;
+            }
+        }
+        return probabilityDictContentToWrite.flush(dictPath,
+                Ver4DictConstants::FREQ_FILE_EXTENSION);
+    } else {
+        return flush(dictPath, Ver4DictConstants::FREQ_FILE_EXTENSION);
+    }
+}
+
+bool ProbabilityDictContent::runGC(
+        const TerminalPositionLookupTable::TerminalIdMap *const terminalIdMap,
+        const ProbabilityDictContent *const originalProbabilityDictContent) {
+    mSize = 0;
+    for (TerminalPositionLookupTable::TerminalIdMap::const_iterator it = terminalIdMap->begin();
+            it != terminalIdMap->end(); ++it) {
+        const ProbabilityEntry probabilityEntry =
+                originalProbabilityDictContent->getProbabilityEntry(it->first);
+        if (!setProbabilityEntry(it->second, &probabilityEntry)) {
+            AKLOGE("Cannot set probability entry in runGC. terminalId: %d", it->second);
+            return false;
+        }
+        mSize++;
+    }
+    return true;
+}
+
+int ProbabilityDictContent::getEntrySize() const {
+    if (mHasHistoricalInfo) {
+        return Ver4DictConstants::FLAGS_IN_PROBABILITY_FILE_SIZE
+                + Ver4DictConstants::PROBABILITY_SIZE
+                + Ver4DictConstants::TIME_STAMP_FIELD_SIZE
+                + Ver4DictConstants::WORD_LEVEL_FIELD_SIZE
+                + Ver4DictConstants::WORD_COUNT_FIELD_SIZE;
+    } else {
+        return Ver4DictConstants::FLAGS_IN_PROBABILITY_FILE_SIZE
+                + Ver4DictConstants::PROBABILITY_SIZE;
+    }
+}
+
+int ProbabilityDictContent::getEntryPos(const int terminalId) const {
+    return terminalId * getEntrySize();
+}
+
+bool ProbabilityDictContent::writeEntry(const ProbabilityEntry *const probabilityEntry,
+        const int entryPos) {
+    BufferWithExtendableBuffer *const bufferToWrite = getWritableBuffer();
+    int writingPos = entryPos;
+    if (!bufferToWrite->writeUintAndAdvancePosition(probabilityEntry->getFlags(),
+            Ver4DictConstants::FLAGS_IN_PROBABILITY_FILE_SIZE, &writingPos)) {
+        AKLOGE("Cannot write flags in probability dict content. pos: %d", writingPos);
+        return false;
+    }
+    if (!bufferToWrite->writeUintAndAdvancePosition(probabilityEntry->getProbability(),
+            Ver4DictConstants::PROBABILITY_SIZE, &writingPos)) {
+        AKLOGE("Cannot write probability in probability dict content. pos: %d", writingPos);
+        return false;
+    }
+    if (mHasHistoricalInfo) {
+        const HistoricalInfo *const historicalInfo = probabilityEntry->getHistoricalInfo();
+        if (!bufferToWrite->writeUintAndAdvancePosition(historicalInfo->getTimeStamp(),
+                Ver4DictConstants::TIME_STAMP_FIELD_SIZE, &writingPos)) {
+            AKLOGE("Cannot write timestamp in probability dict content. pos: %d", writingPos);
+            return false;
+        }
+        if (!bufferToWrite->writeUintAndAdvancePosition(historicalInfo->getLevel(),
+                Ver4DictConstants::WORD_LEVEL_FIELD_SIZE, &writingPos)) {
+            AKLOGE("Cannot write level in probability dict content. pos: %d", writingPos);
+            return false;
+        }
+        if (!bufferToWrite->writeUintAndAdvancePosition(historicalInfo->getCount(),
+                Ver4DictConstants::WORD_COUNT_FIELD_SIZE, &writingPos)) {
+            AKLOGE("Cannot write count in probability dict content. pos: %d", writingPos);
+            return false;
+        }
+    }
+    return true;
+}
+
+} // namespace v401
+} // namespace backward
+} // namespace latinime

native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/probability_dict_content.h

+/*
+ * Copyright (C) 2013, The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * !!!!! DO NOT EDIT THIS FILE !!!!!
+ *
+ * This file was generated from
+ *   suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.h
+ */
+
+#ifndef LATINIME_BACKWARD_V401_PROBABILITY_DICT_CONTENT_H
+#define LATINIME_BACKWARD_V401_PROBABILITY_DICT_CONTENT_H
+
+#include "defines.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v401/content/single_dict_content.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v401/content/terminal_position_lookup_table.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_constants.h"
+#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
+
+namespace latinime {
+namespace backward {
+namespace v401 {
+
+class ProbabilityEntry;
+
+class ProbabilityDictContent : public SingleDictContent {
+ public:
+    ProbabilityDictContent(const char *const dictPath, const bool hasHistoricalInfo,
+            const bool isUpdatable)
+            : SingleDictContent(dictPath, Ver4DictConstants::FREQ_FILE_EXTENSION, isUpdatable),
+              mHasHistoricalInfo(hasHistoricalInfo),
+              mSize(getBuffer()->getTailPosition() / getEntrySize()) {}
+
+    ProbabilityDictContent(const bool hasHistoricalInfo)
+            : mHasHistoricalInfo(hasHistoricalInfo), mSize(0) {}
+
+    const ProbabilityEntry getProbabilityEntry(const int terminalId) const;
+
+    bool setProbabilityEntry(const int terminalId, const ProbabilityEntry *const probabilityEntry);
+
+    bool flushToFile(const char *const dictPath) const;
+
+    bool runGC(const TerminalPositionLookupTable::TerminalIdMap *const terminalIdMap,
+            const ProbabilityDictContent *const originalProbabilityDictContent);
+
+ private:
+    DISALLOW_COPY_AND_ASSIGN(ProbabilityDictContent);
+
+    int getEntrySize() const;
+
+    int getEntryPos(const int terminalId) const;
+
+    bool writeEntry(const ProbabilityEntry *const probabilityEntry, const int entryPos);
+
+    bool mHasHistoricalInfo;
+    int mSize;
+};
+} // namespace v401
+} // namespace backward
+} // namespace latinime
+#endif /* LATINIME_BACKWARD_V401_PROBABILITY_DICT_CONTENT_H */

native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/probability_entry.h

+/*
+ * Copyright (C) 2013, The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * !!!!! DO NOT EDIT THIS FILE !!!!!
+ *
+ * This file was generated from
+ *   suggest/policyimpl/dictionary/structure/v4/content/probability_entry.h
+ */
+
+#ifndef LATINIME_BACKWARD_V401_PROBABILITY_ENTRY_H
+#define LATINIME_BACKWARD_V401_PROBABILITY_ENTRY_H
+
+#include "defines.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_constants.h"
+#include "suggest/policyimpl/dictionary/utils/historical_info.h"
+
+namespace latinime {
+namespace backward {
+namespace v401 {
+
+class ProbabilityEntry {
+ public:
+    ProbabilityEntry(const ProbabilityEntry &probabilityEntry)
+            : mFlags(probabilityEntry.mFlags), mProbability(probabilityEntry.mProbability),
+              mHistoricalInfo(probabilityEntry.mHistoricalInfo) {}
+
+    // Dummy entry
+    ProbabilityEntry()
+            : mFlags(0), mProbability(NOT_A_PROBABILITY), mHistoricalInfo() {}
+
+    // Entry without historical information
+    ProbabilityEntry(const int flags, const int probability)
+            : mFlags(flags), mProbability(probability), mHistoricalInfo() {}
+
+    // Entry with historical information.
+    ProbabilityEntry(const int flags, const int probability,
+            const HistoricalInfo *const historicalInfo)
+            : mFlags(flags), mProbability(probability), mHistoricalInfo(*historicalInfo) {}
+
+    const ProbabilityEntry createEntryWithUpdatedProbability(const int probability) const {
+        return ProbabilityEntry(mFlags, probability, &mHistoricalInfo);
+    }
+
+    const ProbabilityEntry createEntryWithUpdatedHistoricalInfo(
+            const HistoricalInfo *const historicalInfo) const {
+        return ProbabilityEntry(mFlags, mProbability, historicalInfo);
+    }
+
+    bool hasHistoricalInfo() const {
+        return mHistoricalInfo.isValid();
+    }
+
+    int getFlags() const {
+        return mFlags;
+    }
+
+    int getProbability() const {
+        return mProbability;
+    }
+
+    const HistoricalInfo *getHistoricalInfo() const {
+        return &mHistoricalInfo;
+    }
+
+ private:
+    // Copy constructor is public to use this class as a type of return value.
+    DISALLOW_ASSIGNMENT_OPERATOR(ProbabilityEntry);
+
+    const int mFlags;
+    const int mProbability;
+    const HistoricalInfo mHistoricalInfo;
+};
+} // namespace v401
+} // namespace backward
+} // namespace latinime
+#endif /* LATINIME_BACKWARD_V401_PROBABILITY_ENTRY_H */

native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/shortcut_dict_content.cpp

+/*
+ * Copyright (C) 2013, The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * !!!!! DO NOT EDIT THIS FILE !!!!!
+ *
+ * This file was generated from
+ *   suggest/policyimpl/dictionary/structure/v4/content/shortcut_dict_content.cpp
+ */
+
+#include "suggest/policyimpl/dictionary/structure/backward/v401/content/shortcut_dict_content.h"
+
+#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
+
+namespace latinime {
+namespace backward {
+namespace v401 {
+
+void ShortcutDictContent::getShortcutEntryAndAdvancePosition(const int maxCodePointCount,
+        int *const outCodePoint, int *const outCodePointCount, int *const outProbability,
+        bool *const outhasNext, int *const shortcutEntryPos) const {
+    const BufferWithExtendableBuffer *const shortcutListBuffer = getContentBuffer();
+    if (*shortcutEntryPos < 0 || *shortcutEntryPos >=  shortcutListBuffer->getTailPosition()) {
+        AKLOGE("Invalid shortcut entry position. shortcutEntryPos: %d, bufSize: %d",
+                *shortcutEntryPos, shortcutListBuffer->getTailPosition());
+        ASSERT(false);
+        if (outhasNext) {
+            *outhasNext = false;
+        }
+        if (outCodePointCount) {
+            *outCodePointCount = 0;
+        }
+        return;
+    }
+
+    const int shortcutFlags = shortcutListBuffer->readUintAndAdvancePosition(
+            Ver4DictConstants::SHORTCUT_FLAGS_FIELD_SIZE, shortcutEntryPos);
+    if (outProbability) {
+        *outProbability = shortcutFlags & Ver4DictConstants::SHORTCUT_PROBABILITY_MASK;
+    }
+    if (outhasNext) {
+        *outhasNext = shortcutFlags & Ver4DictConstants::SHORTCUT_HAS_NEXT_MASK;
+    }
+    if (outCodePoint && outCodePointCount) {
+        shortcutListBuffer->readCodePointsAndAdvancePosition(
+                maxCodePointCount, outCodePoint, outCodePointCount, shortcutEntryPos);
+    }
+}
+
+int ShortcutDictContent::getShortcutListHeadPos(const int terminalId) const {
+    const SparseTable *const addressLookupTable = getAddressLookupTable();
+    if (!addressLookupTable->contains(terminalId)) {
+        return NOT_A_DICT_POS;
+    }
+    return addressLookupTable->get(terminalId);
+}
+
+bool ShortcutDictContent::flushToFile(const char *const dictPath) const {
+    return flush(dictPath, Ver4DictConstants::SHORTCUT_LOOKUP_TABLE_FILE_EXTENSION,
+            Ver4DictConstants::SHORTCUT_CONTENT_TABLE_FILE_EXTENSION,
+            Ver4DictConstants::SHORTCUT_FILE_EXTENSION);
+}
+
+bool ShortcutDictContent::runGC(
+        const TerminalPositionLookupTable::TerminalIdMap *const terminalIdMap,
+        const ShortcutDictContent *const originalShortcutDictContent) {
+   for (TerminalPositionLookupTable::TerminalIdMap::const_iterator it = terminalIdMap->begin();
+           it != terminalIdMap->end(); ++it) {
+       const int originalShortcutListPos =
+               originalShortcutDictContent->getShortcutListHeadPos(it->first);
+       if (originalShortcutListPos == NOT_A_DICT_POS) {
+           continue;
+       }
+       const int shortcutListPos = getContentBuffer()->getTailPosition();
+       // Copy shortcut list from original content.
+       if (!copyShortcutListFromDictContent(originalShortcutListPos, originalShortcutDictContent,
+               shortcutListPos)) {
+           AKLOGE("Cannot copy shortcut list during GC. original pos: %d, pos: %d",
+                   originalShortcutListPos, shortcutListPos);
+           return false;
+       }
+       // Set shortcut list position to the lookup table.
+       if (!getUpdatableAddressLookupTable()->set(it->second, shortcutListPos)) {
+           AKLOGE("Cannot set shortcut list position. terminal id: %d, pos: %d",
+                   it->second, shortcutListPos);
+           return false;
+       }
+   }
+   return true;
+}
+
+bool ShortcutDictContent::createNewShortcutList(const int terminalId) {
+    const int shortcutListListPos = getContentBuffer()->getTailPosition();
+    return getUpdatableAddressLookupTable()->set(terminalId, shortcutListListPos);
+}
+
+bool ShortcutDictContent::copyShortcutList(const int shortcutListPos, const int toPos) {
+    return copyShortcutListFromDictContent(shortcutListPos, this, toPos);
+}
+
+bool ShortcutDictContent::copyShortcutListFromDictContent(const int shortcutListPos,
+        const ShortcutDictContent *const sourceShortcutDictContent, const int toPos) {
+    bool hasNext = true;
+    int readingPos = shortcutListPos;
+    int writingPos = toPos;
+    int codePoints[MAX_WORD_LENGTH];
+    while (hasNext) {
+        int probability = 0;
+        int codePointCount = 0;
+        sourceShortcutDictContent->getShortcutEntryAndAdvancePosition(MAX_WORD_LENGTH,
+                codePoints, &codePointCount, &probability, &hasNext, &readingPos);
+        if (!writeShortcutEntryAndAdvancePosition(codePoints, codePointCount, probability,
+                hasNext, &writingPos)) {
+            AKLOGE("Cannot write shortcut entry to copy. pos: %d", writingPos);
+            return false;
+        }
+    }
+    return true;
+}
+
+bool ShortcutDictContent::setProbability(const int probability, const int shortcutEntryPos) {
+    BufferWithExtendableBuffer *const shortcutListBuffer = getWritableContentBuffer();
+    const int shortcutFlags = shortcutListBuffer->readUint(
+            Ver4DictConstants::SHORTCUT_FLAGS_FIELD_SIZE, shortcutEntryPos);
+    const bool hasNext = shortcutFlags & Ver4DictConstants::SHORTCUT_HAS_NEXT_MASK;
+    const int shortcutFlagsToWrite = createAndGetShortcutFlags(probability, hasNext);
+    return shortcutListBuffer->writeUint(shortcutFlagsToWrite,
+            Ver4DictConstants::SHORTCUT_FLAGS_FIELD_SIZE, shortcutEntryPos);
+}
+
+bool ShortcutDictContent::writeShortcutEntryAndAdvancePosition(const int *const codePoint,
+        const int codePointCount, const int probability, const bool hasNext,
+        int *const shortcutEntryPos) {
+    BufferWithExtendableBuffer *const shortcutListBuffer = getWritableContentBuffer();
+    const int shortcutFlags = createAndGetShortcutFlags(probability, hasNext);
+    if (!shortcutListBuffer->writeUintAndAdvancePosition(shortcutFlags,
+            Ver4DictConstants::SHORTCUT_FLAGS_FIELD_SIZE, shortcutEntryPos)) {
+        AKLOGE("Cannot write shortcut flags. flags; %x, pos: %d", shortcutFlags, *shortcutEntryPos);
+        return false;
+    }
+    if (!shortcutListBuffer->writeCodePointsAndAdvancePosition(codePoint, codePointCount,
+            true /* writesTerminator */, shortcutEntryPos)) {
+        AKLOGE("Cannot write shortcut target code points. pos: %d", *shortcutEntryPos);
+        return false;
+    }
+    return true;
+}
+
+// Find a shortcut entry that has specified target and return its position.
+int ShortcutDictContent::findShortcutEntryAndGetPos(const int shortcutListPos,
+        const int *const targetCodePointsToFind, const int codePointCount) const {
+    bool hasNext = true;
+    int readingPos = shortcutListPos;
+    int targetCodePoints[MAX_WORD_LENGTH];
+    while (hasNext) {
+        const int entryPos = readingPos;
+        int probability = 0;
+        int targetCodePointCount = 0;
+        getShortcutEntryAndAdvancePosition(MAX_WORD_LENGTH, targetCodePoints, &targetCodePointCount,
+                &probability, &hasNext, &readingPos);
+        if (targetCodePointCount != codePointCount) {
+            continue;
+        }
+        bool matched = true;
+        for (int i = 0; i < codePointCount; ++i) {
+            if (targetCodePointsToFind[i] != targetCodePoints[i]) {
+                matched = false;
+                break;
+            }
+        }
+        if (matched) {
+            return entryPos;
+        }
+    }
+    return NOT_A_DICT_POS;
+}
+
+int ShortcutDictContent::createAndGetShortcutFlags(const int probability,
+        const bool hasNext) const {
+    return (probability & Ver4DictConstants::SHORTCUT_PROBABILITY_MASK)
+            | (hasNext ? Ver4DictConstants::SHORTCUT_HAS_NEXT_MASK : 0);
+}
+
+} // namespace v401
+} // namespace backward
+} // namespace latinime

native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/shortcut_dict_content.h

+/*
+ * Copyright (C) 2013, The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * !!!!! DO NOT EDIT THIS FILE !!!!!
+ *
+ * This file was generated from
+ *   suggest/policyimpl/dictionary/structure/v4/content/shortcut_dict_content.h
+ */
+
+#ifndef LATINIME_BACKWARD_V401_SHORTCUT_DICT_CONTENT_H
+#define LATINIME_BACKWARD_V401_SHORTCUT_DICT_CONTENT_H
+
+#include "defines.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v401/content/sparse_table_dict_content.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v401/content/terminal_position_lookup_table.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_constants.h"
+
+namespace latinime {
+namespace backward {
+namespace v401 {
+
+class ShortcutDictContent : public SparseTableDictContent {
+ public:
+    ShortcutDictContent(const char *const dictPath, const bool isUpdatable)
+            : SparseTableDictContent(dictPath,
+                      Ver4DictConstants::SHORTCUT_LOOKUP_TABLE_FILE_EXTENSION,
+                      Ver4DictConstants::SHORTCUT_CONTENT_TABLE_FILE_EXTENSION,
+                      Ver4DictConstants::SHORTCUT_FILE_EXTENSION, isUpdatable,
+                      Ver4DictConstants::SHORTCUT_ADDRESS_TABLE_BLOCK_SIZE,
+                      Ver4DictConstants::SHORTCUT_ADDRESS_TABLE_DATA_SIZE) {}
+
+    ShortcutDictContent()
+            : SparseTableDictContent(Ver4DictConstants::SHORTCUT_ADDRESS_TABLE_BLOCK_SIZE,
+                      Ver4DictConstants::SHORTCUT_ADDRESS_TABLE_DATA_SIZE) {}
+
+    void getShortcutEntry(const int maxCodePointCount, int *const outCodePoint,
+            int *const outCodePointCount, int *const outProbability, bool *const outhasNext,
+            const int shortcutEntryPos) {
+        int readingPos = shortcutEntryPos;
+        return getShortcutEntryAndAdvancePosition(maxCodePointCount, outCodePoint,
+                outCodePointCount, outProbability, outhasNext, &readingPos);
+    }
+
+    void getShortcutEntryAndAdvancePosition(const int maxCodePointCount,
+            int *const outCodePoint, int *const outCodePointCount, int *const outProbability,
+            bool *const outhasNext, int *const shortcutEntryPos) const;
+
+   // Returns head position of shortcut list for a PtNode specified by terminalId.
+   int getShortcutListHeadPos(const int terminalId) const;
+
+   bool flushToFile(const char *const dictPath) const;
+
+   bool runGC(const TerminalPositionLookupTable::TerminalIdMap *const terminalIdMap,
+           const ShortcutDictContent *const originalShortcutDictContent);
+
+   bool createNewShortcutList(const int terminalId);
+
+   bool copyShortcutList(const int shortcutListPos, const int toPos);
+
+   bool setProbability(const int probability, const int shortcutEntryPos);
+
+   bool writeShortcutEntry(const int *const codePoint, const int codePointCount,
+           const int probability, const bool hasNext, const int shortcutEntryPos) {
+       int writingPos = shortcutEntryPos;
+       return writeShortcutEntryAndAdvancePosition(codePoint, codePointCount, probability,
+               hasNext, &writingPos);
+   }
+
+   bool writeShortcutEntryAndAdvancePosition(const int *const codePoint,
+           const int codePointCount, const int probability, const bool hasNext,
+           int *const shortcutEntryPos);
+
+   int findShortcutEntryAndGetPos(const int shortcutListPos,
+           const int *const targetCodePointsToFind, const int codePointCount) const;
+
+ private:
+    DISALLOW_COPY_AND_ASSIGN(ShortcutDictContent);
+
+    bool copyShortcutListFromDictContent(const int shortcutListPos,
+            const ShortcutDictContent *const sourceShortcutDictContent, const int toPos);
+
+    int createAndGetShortcutFlags(const int probability, const bool hasNext) const;
+};
+} // namespace v401
+} // namespace backward
+} // namespace latinime
+#endif /* LATINIME_BACKWARD_V401_SHORTCUT_DICT_CONTENT_H */

native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/single_dict_content.h

+/*
+ * Copyright (C) 2013, The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * !!!!! DO NOT EDIT THIS FILE !!!!!
+ *
+ * This file was generated from
+ *   suggest/policyimpl/dictionary/structure/v4/content/single_dict_content.h
+ */
+
+#ifndef LATINIME_BACKWARD_V401_SINGLE_DICT_CONTENT_H
+#define LATINIME_BACKWARD_V401_SINGLE_DICT_CONTENT_H
+
+#include "defines.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v401/content/dict_content.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_constants.h"
+#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
+#include "suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h"
+#include "suggest/policyimpl/dictionary/utils/mmapped_buffer.h"
+
+namespace latinime {
+namespace backward {
+namespace v401 {
+
+class SingleDictContent : public DictContent {
+ public:
+    SingleDictContent(const char *const dictPath, const char *const contentFileName,
+            const bool isUpdatable)
+            : mMmappedBuffer(MmappedBuffer::openBuffer(dictPath, contentFileName, isUpdatable)),
+              mExpandableContentBuffer(mMmappedBuffer ? mMmappedBuffer->getBuffer() : nullptr,
+                      mMmappedBuffer ? mMmappedBuffer->getBufferSize() : 0,
+                      BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
+              mIsValid(mMmappedBuffer) {}
+
+    SingleDictContent()
+            : mMmappedBuffer(nullptr),
+              mExpandableContentBuffer(Ver4DictConstants::MAX_DICTIONARY_SIZE), mIsValid(true) {}
+
+    virtual ~SingleDictContent() {}
+
+    virtual bool isValid() const {
+        return mIsValid;
+    }
+
+    bool isNearSizeLimit() const {
+        return mExpandableContentBuffer.isNearSizeLimit();
+    }
+
+ protected:
+    BufferWithExtendableBuffer *getWritableBuffer() {
+        return &mExpandableContentBuffer;
+    }
+
+    const BufferWithExtendableBuffer *getBuffer() const {
+        return &mExpandableContentBuffer;
+    }
+
+    bool flush(const char *const dictPath, const char *const contentFileNameSuffix) const {
+        return DictFileWritingUtils::flushBufferToFileWithSuffix(dictPath,
+                contentFileNameSuffix, &mExpandableContentBuffer);
+    }
+
+ private:
+    DISALLOW_COPY_AND_ASSIGN(SingleDictContent);
+
+    const MmappedBuffer::MmappedBufferPtr mMmappedBuffer;
+    BufferWithExtendableBuffer mExpandableContentBuffer;
+    const bool mIsValid;
+};
+} // namespace v401
+} // namespace backward
+} // namespace latinime
+#endif /* LATINIME_BACKWARD_V401_SINGLE_DICT_CONTENT_H */

native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/sparse_table_dict_content.cpp

+/*
+ * Copyright (C) 2013 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * !!!!! DO NOT EDIT THIS FILE !!!!!
+ *
+ * This file was generated from
+ *   suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.cpp
+ */
+
+#include "suggest/policyimpl/dictionary/structure/backward/v401/content/sparse_table_dict_content.h"
+
+namespace latinime {
+namespace backward {
+namespace v401 {
+
+bool SparseTableDictContent::flush(const char *const dictPath,
+        const char *const lookupTableFileNameSuffix, const char *const addressTableFileNameSuffix,
+        const char *const contentFileNameSuffix) const {
+    if (!DictFileWritingUtils::flushBufferToFileWithSuffix(dictPath, lookupTableFileNameSuffix,
+            &mExpandableLookupTableBuffer)){
+        return false;
+    }
+    if (!DictFileWritingUtils::flushBufferToFileWithSuffix(dictPath, addressTableFileNameSuffix,
+            &mExpandableAddressTableBuffer)) {
+        return false;
+    }
+    if (!DictFileWritingUtils::flushBufferToFileWithSuffix(dictPath, contentFileNameSuffix,
+            &mExpandableContentBuffer)) {
+        return false;
+    }
+    return true;
+}
+
+} // namespace v401
+} // namespace backward
+} // namespace latinime

native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/sparse_table_dict_content.h

+/*
+ * Copyright (C) 2013, The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * !!!!! DO NOT EDIT THIS FILE !!!!!
+ *
+ * This file was generated from
+ *   suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.h
+ */
+
+#ifndef LATINIME_BACKWARD_V401_SPARSE_TABLE_DICT_CONTENT_H
+#define LATINIME_BACKWARD_V401_SPARSE_TABLE_DICT_CONTENT_H
+
+#include "defines.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v401/content/dict_content.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_constants.h"
+#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
+#include "suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h"
+#include "suggest/policyimpl/dictionary/utils/mmapped_buffer.h"
+#include "suggest/policyimpl/dictionary/utils/sparse_table.h"
+
+namespace latinime {
+namespace backward {
+namespace v401 {
+
+// TODO: Support multiple contents.
+class SparseTableDictContent : public DictContent {
+ public:
+    AK_FORCE_INLINE SparseTableDictContent(const char *const dictPath,
+            const char *const lookupTableFileName, const char *const addressTableFileName,
+            const char *const contentFileName, const bool isUpdatable,
+            const int sparseTableBlockSize, const int sparseTableDataSize)
+            : mLookupTableBuffer(
+                      MmappedBuffer::openBuffer(dictPath, lookupTableFileName, isUpdatable)),
+              mAddressTableBuffer(
+                      MmappedBuffer::openBuffer(dictPath, addressTableFileName, isUpdatable)),
+              mContentBuffer(
+                      MmappedBuffer::openBuffer(dictPath, contentFileName, isUpdatable)),
+              mExpandableLookupTableBuffer(
+                      mLookupTableBuffer ? mLookupTableBuffer->getBuffer() : nullptr,
+                      mLookupTableBuffer ? mLookupTableBuffer->getBufferSize() : 0,
+                      BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
+              mExpandableAddressTableBuffer(
+                      mAddressTableBuffer ? mAddressTableBuffer->getBuffer() : nullptr,
+                      mAddressTableBuffer ? mAddressTableBuffer->getBufferSize() : 0,
+                      BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
+              mExpandableContentBuffer(mContentBuffer ? mContentBuffer->getBuffer() : nullptr,
+                      mContentBuffer ? mContentBuffer->getBufferSize() : 0,
+                      BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
+              mAddressLookupTable(&mExpandableLookupTableBuffer, &mExpandableAddressTableBuffer,
+                      sparseTableBlockSize, sparseTableDataSize),
+              mIsValid(mLookupTableBuffer && mAddressTableBuffer && mContentBuffer) {}
+
+    SparseTableDictContent(const int sparseTableBlockSize, const int sparseTableDataSize)
+            : mLookupTableBuffer(), mAddressTableBuffer(), mContentBuffer(),
+              mExpandableLookupTableBuffer(Ver4DictConstants::MAX_DICTIONARY_SIZE),
+              mExpandableAddressTableBuffer(Ver4DictConstants::MAX_DICTIONARY_SIZE),
+              mExpandableContentBuffer(Ver4DictConstants::MAX_DICTIONARY_SIZE),
+              mAddressLookupTable(&mExpandableLookupTableBuffer, &mExpandableAddressTableBuffer,
+                      sparseTableBlockSize, sparseTableDataSize), mIsValid(true) {}
+
+    virtual ~SparseTableDictContent() {}
+
+    virtual bool isValid() const {
+        return mIsValid;
+    }
+
+    bool isNearSizeLimit() const {
+        return mExpandableLookupTableBuffer.isNearSizeLimit()
+                || mExpandableAddressTableBuffer.isNearSizeLimit()
+                || mExpandableContentBuffer.isNearSizeLimit();
+    }
+
+ protected:
+    SparseTable *getUpdatableAddressLookupTable() {
+        return &mAddressLookupTable;
+    }
+
+    const SparseTable *getAddressLookupTable() const {
+        return &mAddressLookupTable;
+    }
+
+    BufferWithExtendableBuffer *getWritableContentBuffer() {
+        return &mExpandableContentBuffer;
+    }
+
+    const BufferWithExtendableBuffer *getContentBuffer() const {
+        return &mExpandableContentBuffer;
+    }
+
+    bool flush(const char *const dictDirPath, const char *const lookupTableFileName,
+            const char *const addressTableFileName, const char *const contentFileName) const;
+
+ private:
+    DISALLOW_IMPLICIT_CONSTRUCTORS(SparseTableDictContent);
+
+    const MmappedBuffer::MmappedBufferPtr mLookupTableBuffer;
+    const MmappedBuffer::MmappedBufferPtr mAddressTableBuffer;
+    const MmappedBuffer::MmappedBufferPtr mContentBuffer;
+    BufferWithExtendableBuffer mExpandableLookupTableBuffer;
+    BufferWithExtendableBuffer mExpandableAddressTableBuffer;
+    BufferWithExtendableBuffer mExpandableContentBuffer;
+    SparseTable mAddressLookupTable;
+    const bool mIsValid;
+};
+} // namespace v401
+} // namespace backward
+} // namespace latinime
+#endif /* LATINIME_BACKWARD_V401_SPARSE_TABLE_DICT_CONTENT_H */

native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/terminal_position_lookup_table.cpp

+/*
+ * Copyright (C) 2013 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * !!!!! DO NOT EDIT THIS FILE !!!!!
+ *
+ * This file was generated from
+ *   suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.cpp
+ */
+
+#include "suggest/policyimpl/dictionary/structure/backward/v401/content/terminal_position_lookup_table.h"
+
+#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_reading_utils.h"
+#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
+
+namespace latinime {
+namespace backward {
+namespace v401 {
+
+int TerminalPositionLookupTable::getTerminalPtNodePosition(const int terminalId) const {
+    if (terminalId < 0 || terminalId >= mSize) {
+        return NOT_A_DICT_POS;
+    }
+    const int terminalPos = getBuffer()->readUint(
+            Ver4DictConstants::TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE, getEntryPos(terminalId));
+    return (terminalPos == Ver4DictConstants::NOT_A_TERMINAL_ADDRESS) ?
+            NOT_A_DICT_POS : terminalPos;
+}
+
+bool TerminalPositionLookupTable::setTerminalPtNodePosition(
+        const int terminalId, const int terminalPtNodePos) {
+    if (terminalId < 0) {
+        return NOT_A_DICT_POS;
+    }
+    while (terminalId >= mSize) {
+        // Write new entry.
+        if (!getWritableBuffer()->writeUint(Ver4DictConstants::NOT_A_TERMINAL_ADDRESS,
+                Ver4DictConstants::TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE, getEntryPos(mSize))) {
+            return false;
+        }
+        mSize++;
+    }
+    const int terminalPos = (terminalPtNodePos != NOT_A_DICT_POS) ?
+            terminalPtNodePos : Ver4DictConstants::NOT_A_TERMINAL_ADDRESS;
+    return getWritableBuffer()->writeUint(terminalPos,
+            Ver4DictConstants::TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE, getEntryPos(terminalId));
+}
+
+bool TerminalPositionLookupTable::flushToFile(const char *const dictPath) const {
+    // If the used buffer size is smaller than the actual buffer size, regenerate the lookup
+    // table and write the new table to the file.
+    if (getEntryPos(mSize) < getBuffer()->getTailPosition()) {
+        TerminalPositionLookupTable lookupTableToWrite;
+        for (int i = 0; i < mSize; ++i) {
+            const int terminalPtNodePosition = getTerminalPtNodePosition(i);
+            if (!lookupTableToWrite.setTerminalPtNodePosition(i, terminalPtNodePosition)) {
+                AKLOGE("Cannot set terminal position to lookupTableToWrite."
+                        " terminalId: %d, position: %d", i, terminalPtNodePosition);
+                return false;
+            }
+        }
+        return lookupTableToWrite.flush(dictPath,
+                Ver4DictConstants::TERMINAL_ADDRESS_TABLE_FILE_EXTENSION);
+    } else {
+        // We can simply use this lookup table because the buffer size has not been
+        // changed.
+        return flush(dictPath, Ver4DictConstants::TERMINAL_ADDRESS_TABLE_FILE_EXTENSION);
+    }
+}
+
+bool TerminalPositionLookupTable::runGCTerminalIds(TerminalIdMap *const terminalIdMap) {
+    int removedEntryCount = 0;
+    int nextNewTerminalId = 0;
+    for (int i = 0; i < mSize; ++i) {
+        const int terminalPos = getBuffer()->readUint(
+                Ver4DictConstants::TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE, getEntryPos(i));
+        if (terminalPos == Ver4DictConstants::NOT_A_TERMINAL_ADDRESS) {
+            // This entry is a garbage.
+            removedEntryCount++;
+        } else {
+            // Give a new terminal id to the entry.
+            if (!getWritableBuffer()->writeUint(terminalPos,
+                    Ver4DictConstants::TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE,
+                    getEntryPos(nextNewTerminalId))) {
+                return false;
+            }
+            // Memorize the mapping to the old terminal id to the new terminal id.
+            terminalIdMap->insert(TerminalIdMap::value_type(i, nextNewTerminalId));
+            nextNewTerminalId++;
+        }
+    }
+    mSize = nextNewTerminalId;
+    return true;
+}
+
+} // namespace v401
+} // namespace backward
+} // namespace latinime

native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/terminal_position_lookup_table.h

+/*
+ * Copyright (C) 2013 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * !!!!! DO NOT EDIT THIS FILE !!!!!
+ *
+ * This file was generated from
+ *   suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.h
+ */
+
+#ifndef LATINIME_BACKWARD_V401_TERMINAL_POSITION_LOOKUP_TABLE_H
+#define LATINIME_BACKWARD_V401_TERMINAL_POSITION_LOOKUP_TABLE_H
+
+#include <unordered_map>
+
+#include "defines.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v401/content/single_dict_content.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_constants.h"
+
+namespace latinime {
+namespace backward {
+namespace v401 {
+
+class TerminalPositionLookupTable : public SingleDictContent {
+ public:
+    typedef std::unordered_map<int, int> TerminalIdMap;
+
+    TerminalPositionLookupTable(const char *const dictPath, const bool isUpdatable)
+            : SingleDictContent(dictPath,
+                      Ver4DictConstants::TERMINAL_ADDRESS_TABLE_FILE_EXTENSION, isUpdatable),
+              mSize(getBuffer()->getTailPosition()
+                      / Ver4DictConstants::TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE) {}
+
+    TerminalPositionLookupTable() : mSize(0) {}
+
+    int getTerminalPtNodePosition(const int terminalId) const;
+
+    bool setTerminalPtNodePosition(const int terminalId, const int terminalPtNodePos);
+
+    int getNextTerminalId() const {
+        return mSize;
+    }
+
+    bool flushToFile(const char *const dictPath) const;
+
+    bool runGCTerminalIds(TerminalIdMap *const terminalIdMap);
+
+ private:
+    DISALLOW_COPY_AND_ASSIGN(TerminalPositionLookupTable);
+
+    int getEntryPos(const int terminalId) const {
+        return terminalId * Ver4DictConstants::TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE;
+    }
+
+    int mSize;
+};
+} // namespace v401
+} // namespace backward
+} // namespace latinime
+#endif // LATINIME_BACKWARD_V401_TERMINAL_POSITION_LOOKUP_TABLE_H

native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/shortcut/ver4_shortcut_list_policy.h

+/*
+ * Copyright (C) 2013 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * !!!!! DO NOT CHANGE THE LOGIC IN THIS FILE !!!!!
+ * Do not edit this file other than updating policy's interface.
+ *
+ * This file was generated from
+ *   suggest/policyimpl/dictionary/structure/v4/shortcut/ver4_shortcut_list_policy.h
+ */
+
+#ifndef LATINIME_BACKWARD_V401_VER4_SHORTCUT_LIST_POLICY_H
+#define LATINIME_BACKWARD_V401_VER4_SHORTCUT_LIST_POLICY_H
+
+#include "defines.h"
+#include "suggest/core/policy/dictionary_shortcuts_structure_policy.h"
+#include "suggest/policyimpl/dictionary/structure/pt_common/shortcut/shortcut_list_reading_utils.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v401/content/shortcut_dict_content.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v401/content/terminal_position_lookup_table.h"
+
+namespace latinime {
+namespace backward {
+namespace v401 {
+
+class Ver4ShortcutListPolicy : public DictionaryShortcutsStructurePolicy {
+ public:
+    Ver4ShortcutListPolicy(ShortcutDictContent *const shortcutDictContent,
+            const TerminalPositionLookupTable *const terminalPositionLookupTable)
+            : mShortcutDictContent(shortcutDictContent) {}
+
+    ~Ver4ShortcutListPolicy() {}
+
+    int getStartPos(const int pos) const {
+        // The first shortcut entry is located at the head position of the shortcut list.
+        return pos;
+    }
+
+    void getNextShortcut(const int maxCodePointCount, int *const outCodePoint,
+            int *const outCodePointCount, bool *const outIsWhitelist, bool *const outHasNext,
+            int *const pos) const {
+        int probability = 0;
+        mShortcutDictContent->getShortcutEntryAndAdvancePosition(maxCodePointCount,
+                outCodePoint, outCodePointCount, &probability, outHasNext, pos);
+        if (outIsWhitelist) {
+            *outIsWhitelist = ShortcutListReadingUtils::isWhitelist(probability);
+        }
+    }
+
+    void skipAllShortcuts(int *const pos) const {
+        // Do nothing because we don't need to skip shortcut lists in ver4 dictionaries.
+    }
+
+    bool addNewShortcut(const int terminalId, const int *const codePoints, const int codePointCount,
+            const int probability) {
+        const int shortcutListPos = mShortcutDictContent->getShortcutListHeadPos(terminalId);
+        if (shortcutListPos == NOT_A_DICT_POS) {
+            // Create shortcut list.
+            if (!mShortcutDictContent->createNewShortcutList(terminalId)) {
+                AKLOGE("Cannot create new shortcut list. terminal id: %d", terminalId);
+                return false;
+            }
+            const int writingPos =  mShortcutDictContent->getShortcutListHeadPos(terminalId);
+            return mShortcutDictContent->writeShortcutEntry(codePoints, codePointCount, probability,
+                    false /* hasNext */, writingPos);
+        }
+        const int entryPos = mShortcutDictContent->findShortcutEntryAndGetPos(shortcutListPos,
+                codePoints, codePointCount);
+        if (entryPos == NOT_A_DICT_POS) {
+            // Add new entry to the shortcut list.
+            // Create new shortcut list.
+            if (!mShortcutDictContent->createNewShortcutList(terminalId)) {
+                AKLOGE("Cannot create new shortcut list. terminal id: %d", terminalId);
+                return false;
+            }
+            int writingPos =  mShortcutDictContent->getShortcutListHeadPos(terminalId);
+            if (!mShortcutDictContent->writeShortcutEntryAndAdvancePosition(codePoints,
+                    codePointCount, probability, true /* hasNext */, &writingPos)) {
+                AKLOGE("Cannot write shortcut entry. terminal id: %d, pos: %d", terminalId,
+                        writingPos);
+                return false;
+            }
+            return mShortcutDictContent->copyShortcutList(shortcutListPos, writingPos);
+        }
+        // Overwrite existing entry.
+        bool hasNext = false;
+        mShortcutDictContent->getShortcutEntry(MAX_WORD_LENGTH, 0 /* outCodePoint */,
+                0 /* outCodePointCount */ , 0 /* probability */, &hasNext, entryPos);
+        if (!mShortcutDictContent->writeShortcutEntry(codePoints,
+                codePointCount, probability, hasNext, entryPos)) {
+            AKLOGE("Cannot overwrite shortcut entry. terminal id: %d, pos: %d", terminalId,
+                    entryPos);
+            return false;
+        }
+        return true;
+    }
+
+ private:
+    DISALLOW_IMPLICIT_CONSTRUCTORS(Ver4ShortcutListPolicy);
+
+    ShortcutDictContent *const mShortcutDictContent;
+};
+} // namespace v401
+} // namespace backward
+} // namespace latinime
+#endif // LATINIME_BACKWARD_V401_VER4_SHORTCUT_LIST_POLICY_H