diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.cpp
index 2198a13c931112518ddacc3c448f60912314bfe7..f91dd0e5651fa80218335bcaaa47de35075a8194 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.cpp
@@ -250,7 +250,7 @@ void DynamicPatriciaTriePolicy::flush(const char *const filePath) {
}
DynamicPatriciaTrieWritingHelper writingHelper(&mBufferWithExtendableBuffer,
&mBigramListPolicy, &mShortcutListPolicy);
- writingHelper.writeToDictFile(filePath, mBuffer->getBuffer(), mHeaderPolicy.getSize());
+ writingHelper.writeToDictFile(filePath, &mHeaderPolicy);
}
void DynamicPatriciaTriePolicy::flushWithGC(const char *const filePath) {
diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h
index 2cbb0ff3be4c49fa15f0b165810d40541092e68d..ebe1f3212a060e31d429acbb2861902c54ad553b 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h
@@ -33,7 +33,7 @@ class DicNodeVector;
class DynamicPatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
public:
DynamicPatriciaTriePolicy(const MmappedBuffer *const buffer)
- : mBuffer(buffer), mHeaderPolicy(mBuffer->getBuffer()),
+ : mBuffer(buffer), mHeaderPolicy(mBuffer->getBuffer(), buffer->getBufferSize()),
mBufferWithExtendableBuffer(mBuffer->getBuffer() + mHeaderPolicy.getSize(),
mBuffer->getBufferSize() - mHeaderPolicy.getSize()),
mShortcutListPolicy(&mBufferWithExtendableBuffer),
diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.cpp b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.cpp
index a67c0d94ae30abe6c7426f24c052628e6a3a5a3d..31178fb5cf813ed9f1f4463d1ac987f5078da5b0 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.cpp
@@ -24,6 +24,7 @@
#include "suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_helper.h"
#include "suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_utils.h"
#include "suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_utils.h"
+#include "suggest/policyimpl/dictionary/header/header_policy.h"
#include "suggest/policyimpl/dictionary/patricia_trie_reading_utils.h"
#include "suggest/policyimpl/dictionary/shortcut/dynamic_shortcut_list_policy.h"
@@ -137,7 +138,11 @@ bool DynamicPatriciaTrieWritingHelper::removeBigramWords(const int word0Pos, con
}
void DynamicPatriciaTrieWritingHelper::writeToDictFile(const char *const fileName,
- const uint8_t *const headerBuf, const int headerSize) {
+ const HeaderPolicy *const headerPolicy) {
+ BufferWithExtendableBuffer headerBuffer(0 /* originalBuffer */, 0 /* originalBufferSize */);
+ if (!headerPolicy->writeHeaderToBuffer(&headerBuffer, false /* updatesLastUpdatedTime */)) {
+ return;
+ }
const int tmpFileNameBufSize = strlen(fileName)
+ strlen(TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE) + 1;
char tmpFileName[tmpFileNameBufSize];
@@ -148,7 +153,8 @@ void DynamicPatriciaTrieWritingHelper::writeToDictFile(const char *const fileNam
return;
}
// Write header.
- if (fwrite(headerBuf, headerSize, 1, file) < 1) {
+ if (fwrite(headerBuffer.getBuffer(true /* usesAdditionalBuffer */),
+ headerBuffer.getTailPosition(), 1, file) < 1) {
fclose(file);
remove(tmpFileName);
return;
diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h
index faf7a4e1b4891d87304dd90e2bd0fb3c52cdd189..219ea98574687c7c4ef9d14ce9ce221d898dc560 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h
@@ -28,6 +28,7 @@ class DynamicBigramListPolicy;
class DynamicPatriciaTrieNodeReader;
class DynamicPatriciaTrieReadingHelper;
class DynamicShortcutListPolicy;
+class HeaderPolicy;
class DynamicPatriciaTrieWritingHelper {
public:
@@ -48,8 +49,9 @@ class DynamicPatriciaTrieWritingHelper {
// Remove a bigram relation from word0Pos to word1Pos.
bool removeBigramWords(const int word0Pos, const int word1Pos);
- void writeToDictFile(const char *const fileName, const uint8_t *const headerBuf,
- const int headerSize);
+ void writeToDictFile(const char *const fileName, const HeaderPolicy *const headerPolicy);
+
+ void writeToDictFileWithGC(const char *const fileName, const HeaderPolicy *const headerPolicy);
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicPatriciaTrieWritingHelper);
diff --git a/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.cpp
index 196da5c975a617685fd6aa44be8454db49a22bdf..47ace23a1aaff50a402acff5567c3fc6f4668343 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.cpp
@@ -17,6 +17,8 @@
#include "suggest/policyimpl/dictionary/header/header_policy.h"
#include <cstddef>
+#include <cstdio>
+#include <ctime>
namespace latinime {
@@ -36,7 +38,7 @@ void HeaderPolicy::readHeaderValueOrQuestionMark(const char *const key, int *out
}
std::vector<int> keyCodePointVector;
insertCharactersIntoVector(key, &keyCodePointVector);
- HeaderReadingUtils::AttributeMap::const_iterator it = mAttributeMap.find(keyCodePointVector);
+ HeaderReadWriteUtils::AttributeMap::const_iterator it = mAttributeMap.find(keyCodePointVector);
if (it == mAttributeMap.end()) {
// The key was not found.
outValue[0] = '?';
@@ -85,7 +87,7 @@ int HeaderPolicy::readLastUpdatedTime() const {
bool HeaderPolicy::getAttributeValueAsInt(const char *const key, int *const outValue) const {
std::vector<int> keyVector;
insertCharactersIntoVector(key, &keyVector);
- HeaderReadingUtils::AttributeMap::const_iterator it = mAttributeMap.find(keyVector);
+ HeaderReadWriteUtils::AttributeMap::const_iterator it = mAttributeMap.find(keyVector);
if (it == mAttributeMap.end()) {
// The key was not found.
return false;
@@ -94,10 +96,56 @@ bool HeaderPolicy::getAttributeValueAsInt(const char *const key, int *const outV
return true;
}
-/* static */ HeaderReadingUtils::AttributeMap HeaderPolicy::createAttributeMapAndReadAllAttributes(
- const uint8_t *const dictBuf) {
- HeaderReadingUtils::AttributeMap attributeMap;
- HeaderReadingUtils::fetchAllHeaderAttributes(dictBuf, &attributeMap);
+bool HeaderPolicy::writeHeaderToBuffer(BufferWithExtendableBuffer *const bufferToWrite,
+ const bool updatesLastUpdatedTime) const {
+ int writingPos = 0;
+ if (!HeaderReadWriteUtils::writeDictionaryVersion(bufferToWrite, mDictFormatVersion,
+ &writingPos)) {
+ return false;
+ }
+ if (!HeaderReadWriteUtils::writeDictionaryFlags(bufferToWrite, mDictionaryFlags,
+ &writingPos)) {
+ return false;
+ }
+ // Temporarily writes a dummy header size.
+ int headerSizeFieldPos = writingPos;
+ if (!HeaderReadWriteUtils::writeDictionaryHeaderSize(bufferToWrite, 0 /* size */,
+ &writingPos)) {
+ return false;
+ }
+ if (updatesLastUpdatedTime) {
+ // Set current time as a last updated time.
+ HeaderReadWriteUtils::AttributeMap attributeMapTowrite(mAttributeMap);
+ std::vector<int> updatedTimekey;
+ insertCharactersIntoVector(LAST_UPDATED_TIME_KEY, &updatedTimekey);
+ const time_t currentTime = time(NULL);
+ std::vector<int> updatedTimeValue;
+ char charBuf[LARGEST_INT_DIGIT_COUNT + 1];
+ snprintf(charBuf, LARGEST_INT_DIGIT_COUNT + 1, "%ld", currentTime);
+ insertCharactersIntoVector(charBuf, &updatedTimeValue);
+ attributeMapTowrite[updatedTimekey] = updatedTimeValue;
+ if (!HeaderReadWriteUtils::writeHeaderAttributes(bufferToWrite, &attributeMapTowrite,
+ &writingPos)) {
+ return false;
+ }
+ } else {
+ if (!HeaderReadWriteUtils::writeHeaderAttributes(bufferToWrite, &mAttributeMap,
+ &writingPos)) {
+ return false;
+ }
+ }
+ // Writes an actual header size.
+ if (!HeaderReadWriteUtils::writeDictionaryHeaderSize(bufferToWrite, writingPos,
+ &headerSizeFieldPos)) {
+ return false;
+ }
+ return true;
+}
+
+/* static */ HeaderReadWriteUtils::AttributeMap
+ HeaderPolicy::createAttributeMapAndReadAllAttributes(const uint8_t *const dictBuf) {
+ HeaderReadWriteUtils::AttributeMap attributeMap;
+ HeaderReadWriteUtils::fetchAllHeaderAttributes(dictBuf, &attributeMap);
return attributeMap;
}
diff --git a/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h b/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h
index 930b475c706832b5a6b05de91a8c0294525b7bb3..cdad7de4d351f8897eb8cd457478debbb0bca797 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h
@@ -23,14 +23,17 @@
#include "defines.h"
#include "suggest/core/policy/dictionary_header_structure_policy.h"
#include "suggest/policyimpl/dictionary/header/header_reading_utils.h"
+#include "suggest/policyimpl/dictionary/utils/format_utils.h"
namespace latinime {
class HeaderPolicy : public DictionaryHeaderStructurePolicy {
public:
- explicit HeaderPolicy(const uint8_t *const dictBuf)
- : mDictBuf(dictBuf), mDictionaryFlags(HeaderReadingUtils::getFlags(dictBuf)),
- mSize(HeaderReadingUtils::getHeaderSize(dictBuf)),
+ explicit HeaderPolicy(const uint8_t *const dictBuf, const int dictSize)
+ : mDictBuf(dictBuf),
+ mDictFormatVersion(FormatUtils::detectFormatVersion(dictBuf, dictSize)),
+ mDictionaryFlags(HeaderReadWriteUtils::getFlags(dictBuf)),
+ mSize(HeaderReadWriteUtils::getHeaderSize(dictBuf)),
mAttributeMap(createAttributeMapAndReadAllAttributes(mDictBuf)),
mMultiWordCostMultiplier(readMultipleWordCostMultiplier()),
mUsesForgettingCurve(readUsesForgettingCurveFlag()),
@@ -43,16 +46,15 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
}
AK_FORCE_INLINE bool supportsDynamicUpdate() const {
- return HeaderReadingUtils::supportsDynamicUpdate(mDictionaryFlags);
+ return HeaderReadWriteUtils::supportsDynamicUpdate(mDictionaryFlags);
}
AK_FORCE_INLINE bool requiresGermanUmlautProcessing() const {
- return HeaderReadingUtils::requiresGermanUmlautProcessing(mDictionaryFlags);
+ return HeaderReadWriteUtils::requiresGermanUmlautProcessing(mDictionaryFlags);
}
AK_FORCE_INLINE bool requiresFrenchLigatureProcessing() const {
- return HeaderReadingUtils::requiresFrenchLigatureProcessing(
- mDictionaryFlags);
+ return HeaderReadWriteUtils::requiresFrenchLigatureProcessing(mDictionaryFlags);
}
AK_FORCE_INLINE float getMultiWordCostMultiplier() const {
@@ -70,6 +72,9 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
void readHeaderValueOrQuestionMark(const char *const key,
int *outValue, int outValueSize) const;
+ bool writeHeaderToBuffer(BufferWithExtendableBuffer *const bufferToWrite,
+ const bool updatesLastUpdatedTime) const;
+
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(HeaderPolicy);
@@ -80,9 +85,10 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
static const float MULTIPLE_WORD_COST_MULTIPLIER_SCALE;
const uint8_t *const mDictBuf;
- const HeaderReadingUtils::DictionaryFlags mDictionaryFlags;
+ const FormatUtils::FORMAT_VERSION mDictFormatVersion;
+ const HeaderReadWriteUtils::DictionaryFlags mDictionaryFlags;
const int mSize;
- HeaderReadingUtils::AttributeMap mAttributeMap;
+ HeaderReadWriteUtils::AttributeMap mAttributeMap;
const float mMultiWordCostMultiplier;
const bool mUsesForgettingCurve;
const int mLastUpdatedTime;
@@ -95,7 +101,7 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
bool getAttributeValueAsInt(const char *const key, int *const outValue) const;
- static HeaderReadingUtils::AttributeMap createAttributeMapAndReadAllAttributes(
+ static HeaderReadWriteUtils::AttributeMap createAttributeMapAndReadAllAttributes(
const uint8_t *const dictBuf);
static int parseIntAttributeValue(const std::vector<int> *const attributeValue);
diff --git a/native/jni/src/suggest/policyimpl/dictionary/header/header_reading_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/header/header_reading_utils.cpp
index 186c043c177d41051e6697947f7404035ca848f7..705355db3322af34094d2b0fe822d7983a1539a5 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/header/header_reading_utils.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/header/header_reading_utils.cpp
@@ -19,43 +19,44 @@
#include <vector>
#include "defines.h"
+#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
#include "suggest/policyimpl/dictionary/utils/byte_array_utils.h"
namespace latinime {
-const int HeaderReadingUtils::MAX_ATTRIBUTE_KEY_LENGTH = 256;
-const int HeaderReadingUtils::MAX_ATTRIBUTE_VALUE_LENGTH = 256;
+const int HeaderReadWriteUtils::MAX_ATTRIBUTE_KEY_LENGTH = 256;
+const int HeaderReadWriteUtils::MAX_ATTRIBUTE_VALUE_LENGTH = 256;
-const int HeaderReadingUtils::HEADER_MAGIC_NUMBER_SIZE = 4;
-const int HeaderReadingUtils::HEADER_DICTIONARY_VERSION_SIZE = 2;
-const int HeaderReadingUtils::HEADER_FLAG_SIZE = 2;
-const int HeaderReadingUtils::HEADER_SIZE_FIELD_SIZE = 4;
+const int HeaderReadWriteUtils::HEADER_MAGIC_NUMBER_SIZE = 4;
+const int HeaderReadWriteUtils::HEADER_DICTIONARY_VERSION_SIZE = 2;
+const int HeaderReadWriteUtils::HEADER_FLAG_SIZE = 2;
+const int HeaderReadWriteUtils::HEADER_SIZE_FIELD_SIZE = 4;
-const HeaderReadingUtils::DictionaryFlags HeaderReadingUtils::NO_FLAGS = 0;
+const HeaderReadWriteUtils::DictionaryFlags HeaderReadWriteUtils::NO_FLAGS = 0;
// Flags for special processing
// Those *must* match the flags in makedict (FormatSpec#*_PROCESSING_FLAG) or
// something very bad (like, the apocalypse) will happen. Please update both at the same time.
-const HeaderReadingUtils::DictionaryFlags
- HeaderReadingUtils::GERMAN_UMLAUT_PROCESSING_FLAG = 0x1;
-const HeaderReadingUtils::DictionaryFlags
- HeaderReadingUtils::SUPPORTS_DYNAMIC_UPDATE_FLAG = 0x2;
-const HeaderReadingUtils::DictionaryFlags
- HeaderReadingUtils::FRENCH_LIGATURE_PROCESSING_FLAG = 0x4;
+const HeaderReadWriteUtils::DictionaryFlags
+ HeaderReadWriteUtils::GERMAN_UMLAUT_PROCESSING_FLAG = 0x1;
+const HeaderReadWriteUtils::DictionaryFlags
+ HeaderReadWriteUtils::SUPPORTS_DYNAMIC_UPDATE_FLAG = 0x2;
+const HeaderReadWriteUtils::DictionaryFlags
+ HeaderReadWriteUtils::FRENCH_LIGATURE_PROCESSING_FLAG = 0x4;
-/* static */ int HeaderReadingUtils::getHeaderSize(const uint8_t *const dictBuf) {
+/* static */ int HeaderReadWriteUtils::getHeaderSize(const uint8_t *const dictBuf) {
// See the format of the header in the comment in
// BinaryDictionaryFormatUtils::detectFormatVersion()
return ByteArrayUtils::readUint32(dictBuf, HEADER_MAGIC_NUMBER_SIZE
+ HEADER_DICTIONARY_VERSION_SIZE + HEADER_FLAG_SIZE);
}
-/* static */ HeaderReadingUtils::DictionaryFlags
- HeaderReadingUtils::getFlags(const uint8_t *const dictBuf) {
+/* static */ HeaderReadWriteUtils::DictionaryFlags
+ HeaderReadWriteUtils::getFlags(const uint8_t *const dictBuf) {
return ByteArrayUtils::readUint16(dictBuf,
HEADER_MAGIC_NUMBER_SIZE + HEADER_DICTIONARY_VERSION_SIZE);
}
-/* static */ void HeaderReadingUtils::fetchAllHeaderAttributes(const uint8_t *const dictBuf,
+/* static */ void HeaderReadWriteUtils::fetchAllHeaderAttributes(const uint8_t *const dictBuf,
AttributeMap *const headerAttributes) {
const int headerSize = getHeaderSize(dictBuf);
int pos = getHeaderOptionsPosition();
@@ -78,4 +79,53 @@ const HeaderReadingUtils::DictionaryFlags
}
}
+/* static */ bool HeaderReadWriteUtils::writeDictionaryVersion(
+ BufferWithExtendableBuffer *const buffer, const FormatUtils::FORMAT_VERSION version,
+ int *const writingPos) {
+ if (!buffer->writeUintAndAdvancePosition(FormatUtils::MAGIC_NUMBER, HEADER_MAGIC_NUMBER_SIZE,
+ writingPos)) {
+ return false;
+ }
+ switch (version) {
+ case FormatUtils::VERSION_2:
+ // Version 2 dictionary writing is not supported.
+ return false;
+ case FormatUtils::VERSION_3:
+ return buffer->writeUintAndAdvancePosition(3 /* data */,
+ HEADER_DICTIONARY_VERSION_SIZE, writingPos);
+ default:
+ return false;
+ }
+}
+
+/* static */ bool HeaderReadWriteUtils::writeDictionaryFlags(
+ BufferWithExtendableBuffer *const buffer, const DictionaryFlags flags,
+ int *const writingPos) {
+ return buffer->writeUintAndAdvancePosition(flags, HEADER_FLAG_SIZE, writingPos);
+}
+
+/* static */ bool HeaderReadWriteUtils::writeDictionaryHeaderSize(
+ BufferWithExtendableBuffer *const buffer, const int size, int *const writingPos) {
+ return buffer->writeUintAndAdvancePosition(size, HEADER_SIZE_FIELD_SIZE, writingPos);
+}
+
+/* static */ bool HeaderReadWriteUtils::writeHeaderAttributes(
+ BufferWithExtendableBuffer *const buffer, const AttributeMap *const headerAttributes,
+ int *const writingPos) {
+ for (AttributeMap::const_iterator it = headerAttributes->begin();
+ it != headerAttributes->end(); ++it) {
+ // Write a key.
+ if (!buffer->writeCodePointsAndAdvancePosition(&(it->first.at(0)), it->first.size(),
+ true /* writesTerminator */, writingPos)) {
+ return false;
+ }
+ // Write a value.
+ if (!buffer->writeCodePointsAndAdvancePosition(&(it->second.at(0)), it->second.size(),
+ true /* writesTerminator */, writingPos)) {
+ return false;
+ }
+ }
+ return true;
+}
+
} // namespace latinime
diff --git a/native/jni/src/suggest/policyimpl/dictionary/header/header_reading_utils.h b/native/jni/src/suggest/policyimpl/dictionary/header/header_reading_utils.h
index 5716198fb94f97a9256fb68944920dceff7fbe8b..4b4e2205aa6fa2776b54671bc6f203baee0bc1ce 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/header/header_reading_utils.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/header/header_reading_utils.h
@@ -22,10 +22,14 @@
#include <vector>
#include "defines.h"
+#include "suggest/policyimpl/dictionary/utils/format_utils.h"
namespace latinime {
-class HeaderReadingUtils {
+class BufferWithExtendableBuffer;
+
+// TODO: Change this file name to header_read_write_utils.h.
+class HeaderReadWriteUtils {
public:
typedef uint16_t DictionaryFlags;
typedef std::map<std::vector<int>, std::vector<int> > AttributeMap;
@@ -54,8 +58,20 @@ class HeaderReadingUtils {
static void fetchAllHeaderAttributes(const uint8_t *const dictBuf,
AttributeMap *const headerAttributes);
+ static bool writeDictionaryVersion(BufferWithExtendableBuffer *const buffer,
+ const FormatUtils::FORMAT_VERSION version, int *const writingPos);
+
+ static bool writeDictionaryFlags(BufferWithExtendableBuffer *const buffer,
+ const DictionaryFlags flags, int *const writingPos);
+
+ static bool writeDictionaryHeaderSize(BufferWithExtendableBuffer *const buffer,
+ const int size, int *const writingPos);
+
+ static bool writeHeaderAttributes(BufferWithExtendableBuffer *const buffer,
+ const AttributeMap *const headerAttributes, int *const writingPos);
+
private:
- DISALLOW_IMPLICIT_CONSTRUCTORS(HeaderReadingUtils);
+ DISALLOW_IMPLICIT_CONSTRUCTORS(HeaderReadWriteUtils);
static const int MAX_ATTRIBUTE_KEY_LENGTH;
static const int MAX_ATTRIBUTE_VALUE_LENGTH;
diff --git a/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.h b/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.h
index cee3e4ab2d05e356c70cc84e6398fd06be9e3585..697d0159cd55db599547349cab4f1b6a823077b5 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.h
@@ -34,7 +34,7 @@ class DicNodeVector;
class PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
public:
PatriciaTriePolicy(const MmappedBuffer *const buffer)
- : mBuffer(buffer), mHeaderPolicy(mBuffer->getBuffer()),
+ : mBuffer(buffer), mHeaderPolicy(mBuffer->getBuffer(), buffer->getBufferSize()),
mDictRoot(mBuffer->getBuffer() + mHeaderPolicy.getSize()),
mBigramListPolicy(mDictRoot), mShortcutListPolicy(mDictRoot) {}
diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/format_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/utils/format_utils.cpp
index 3796c7b7bd79c8511ec623a7aafc6c68a2a5ee04..1d77d5c27b282335226d79ba0f4ac02da246d93e 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/utils/format_utils.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/utils/format_utils.cpp
@@ -20,20 +20,10 @@
namespace latinime {
-/**
- * Dictionary size
- */
-// Any file smaller than this is not a dictionary.
-const int FormatUtils::DICTIONARY_MINIMUM_SIZE = 4;
+const uint32_t FormatUtils::MAGIC_NUMBER = 0x9BC13AFE;
-/**
- * Format versions
- */
-// 32 bit magic number is stored at the beginning of the dictionary header to reject unsupported
-// or obsolete dictionary formats.
-const uint32_t FormatUtils::HEADER_VERSION_2_MAGIC_NUMBER = 0x9BC13AFE;
-// Magic number (4 bytes), version (2 bytes), options (2 bytes), header size (4 bytes) = 12
-const int FormatUtils::HEADER_VERSION_2_MINIMUM_SIZE = 12;
+// Magic number (4 bytes), version (2 bytes), flags (2 bytes), header size (4 bytes) = 12
+const int FormatUtils::DICTIONARY_MINIMUM_SIZE = 12;
/* static */ FormatUtils::FORMAT_VERSION FormatUtils::detectFormatVersion(
const uint8_t *const dict, const int dictSize) {
@@ -45,16 +35,10 @@ const int FormatUtils::HEADER_VERSION_2_MINIMUM_SIZE = 12;
}
const uint32_t magicNumber = ByteArrayUtils::readUint32(dict, 0);
switch (magicNumber) {
- case HEADER_VERSION_2_MAGIC_NUMBER:
- // Version 2 header are at least 12 bytes long.
- // If this header has the version 2 magic number but is less than 12 bytes long,
- // then it's an unknown format and we need to avoid confidently reading the next bytes.
- if (dictSize < HEADER_VERSION_2_MINIMUM_SIZE) {
- return UNKNOWN_VERSION;
- }
+ case MAGIC_NUMBER:
// Version 2 header is as follows:
// Magic number (4 bytes) 0x9B 0xC1 0x3A 0xFE
- // Version number (2 bytes)
+ // Dictionary format version number (2 bytes)
// Options (2 bytes)
// Header size (4 bytes) : integer, big endian
if (ByteArrayUtils::readUint16(dict, 4) == 2) {
diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/format_utils.h b/native/jni/src/suggest/policyimpl/dictionary/utils/format_utils.h
index f84321577d146eea77fc9f45b8e004acf2a5ae0e..79ed0de294d824be7de048d454d3a4c9bbc40ca7 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/utils/format_utils.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/utils/format_utils.h
@@ -34,14 +34,16 @@ class FormatUtils {
UNKNOWN_VERSION
};
+ // 32 bit magic number is stored at the beginning of the dictionary header to reject
+ // unsupported or obsolete dictionary formats.
+ static const uint32_t MAGIC_NUMBER;
+
static FORMAT_VERSION detectFormatVersion(const uint8_t *const dict, const int dictSize);
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(FormatUtils);
static const int DICTIONARY_MINIMUM_SIZE;
- static const uint32_t HEADER_VERSION_2_MAGIC_NUMBER;
- static const int HEADER_VERSION_2_MINIMUM_SIZE;
};
} // namespace latinime
#endif /* LATINIME_FORMAT_UTILS_H */
diff --git a/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java b/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java
index d8105ba386a9f8265c1c5881d7b0144be515c84c..00d76c990e5cd8236f11f9622df99cdf98eb0657 100644
--- a/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java
+++ b/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java
@@ -358,8 +358,8 @@ public class BinaryDictionaryTests extends AndroidTestCase {
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
- assertEquals(-1, binaryDictionary.getFrequency("aaa"));
- assertEquals(-1, binaryDictionary.getFrequency("abcd"));
+ assertEquals(Dictionary.NOT_A_PROBABILITY, binaryDictionary.getFrequency("aaa"));
+ assertEquals(Dictionary.NOT_A_PROBABILITY, binaryDictionary.getFrequency("abcd"));
binaryDictionary.addUnigramWord("aaa", probability);
binaryDictionary.addUnigramWord("abcd", probability);