From 2f0c1253e288d0670416bf8fc45b77962e68e250 Mon Sep 17 00:00:00 2001
From: Keisuke Kuroyanagi <ksk@google.com>
Date: Tue, 27 Aug 2013 17:48:04 +0900
Subject: [PATCH] Use extendable buffer for reading bigram and shortcut.
Bug: 6669677
Change-Id: I4017c0ecc5d8baff1b16ff370d3eec85de9a0f9f
---
.../bigram/dynamic_bigram_list_policy.h | 80 ++++++++++++++++
.../shortcut/dynamic_shortcut_list_policy.h | 94 +++++++++++++++++++
.../shortcut/shortcut_list_reading_utils.h | 4 +
.../dictionary/utils/extendable_buffer.h | 4 +-
4 files changed, 180 insertions(+), 2 deletions(-)
create mode 100644 native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.h
create mode 100644 native/jni/src/suggest/policyimpl/dictionary/shortcut/dynamic_shortcut_list_policy.h
diff --git a/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.h b/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.h
new file mode 100644
index 0000000000..513878e1d6
--- /dev/null
+++ b/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.h
@@ -0,0 +1,80 @@
+/*
+ * Copyright (C) 2013 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LATINIME_DYNAMIC_BIGRAM_LIST_POLICY_H
+#define LATINIME_DYNAMIC_BIGRAM_LIST_POLICY_H
+
+#include <stdint.h>
+
+#include "defines.h"
+#include "suggest/core/policy/dictionary_bigrams_structure_policy.h"
+#include "suggest/policyimpl/dictionary/bigram/bigram_list_reading_utils.h"
+#include "suggest/policyimpl/dictionary/utils/extendable_buffer.h"
+
+namespace latinime {
+
+/*
+ * This is a dynamic version of BigramListPolicy and supports an additional buffer.
+ */
+class DynamicBigramListPolicy : public DictionaryBigramsStructurePolicy {
+ public:
+ DynamicBigramListPolicy(const uint8_t *const bigramsBuf, const int bufSize,
+ const ExtendableBuffer *const additionalBuffer)
+ : mDictRoot(bigramsBuf), mBufSize(bufSize), mAdditionalBuffer(additionalBuffer) {}
+
+ ~DynamicBigramListPolicy() {}
+
+ void getNextBigram(int *const outBigramPos, int *const outProbability, bool *const outHasNext,
+ int *const pos) const {
+ const bool usesAdditionalBuffer = *pos >= mBufSize;
+ const uint8_t *const buffer = (usesAdditionalBuffer) ?
+ mAdditionalBuffer->getBuffer() : mDictRoot;
+ if (usesAdditionalBuffer) {
+ *pos -= mBufSize;
+ }
+ const BigramListReadingUtils::BigramFlags flags =
+ BigramListReadingUtils::getFlagsAndForwardPointer(buffer, pos);
+ *outBigramPos = BigramListReadingUtils::getBigramAddressAndForwardPointer(
+ buffer, flags, pos);
+ if (usesAdditionalBuffer) {
+ *outBigramPos += mBufSize;
+ }
+ *outProbability = BigramListReadingUtils::getProbabilityFromFlags(flags);
+ *outHasNext = BigramListReadingUtils::hasNext(flags);
+ if (usesAdditionalBuffer) {
+ *pos += mBufSize;
+ }
+ }
+
+ void skipAllBigrams(int *const pos) const {
+ if (*pos >= mBufSize) {
+ *pos -= mBufSize;
+ BigramListReadingUtils::skipExistingBigrams(mAdditionalBuffer->getBuffer(), pos);
+ *pos += mBufSize;
+ } else {
+ BigramListReadingUtils::skipExistingBigrams(mDictRoot, pos);
+ }
+ }
+
+ private:
+ DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicBigramListPolicy);
+
+ const uint8_t *const mDictRoot;
+ const int mBufSize;
+ const ExtendableBuffer *const mAdditionalBuffer;
+};
+} // namespace latinime
+#endif // LATINIME_DYNAMIC_BIGRAM_LIST_POLICY_H
diff --git a/native/jni/src/suggest/policyimpl/dictionary/shortcut/dynamic_shortcut_list_policy.h b/native/jni/src/suggest/policyimpl/dictionary/shortcut/dynamic_shortcut_list_policy.h
new file mode 100644
index 0000000000..fdaf18f7c1
--- /dev/null
+++ b/native/jni/src/suggest/policyimpl/dictionary/shortcut/dynamic_shortcut_list_policy.h
@@ -0,0 +1,94 @@
+/*
+ * Copyright (C) 2013 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LATINIME_DYNAMIC_SHORTCUT_LIST_POLICY_H
+#define LATINIME_DYNAMIC_SHORTCUT_LIST_POLICY_H
+
+#include <stdint.h>
+
+#include "defines.h"
+#include "suggest/core/policy/dictionary_shortcuts_structure_policy.h"
+#include "suggest/policyimpl/dictionary/shortcut/shortcut_list_reading_utils.h"
+#include "suggest/policyimpl/dictionary/utils/extendable_buffer.h"
+
+namespace latinime {
+
+/*
+ * This is a dynamic version of ShortcutListPolicy and supports an additional buffer.
+ */
+class DynamicShortcutListPolicy : public DictionaryShortcutsStructurePolicy {
+ public:
+ DynamicShortcutListPolicy(const uint8_t *const shortcutBuf, const int bufSize,
+ const ExtendableBuffer *const additionalBuffer)
+ : mShortcutsBuf(shortcutBuf), mBufSize(bufSize), mAdditionalBuffer(additionalBuffer) {}
+
+ ~DynamicShortcutListPolicy() {}
+
+ int getStartPos(const int pos) const {
+ if (pos == NOT_A_DICT_POS) {
+ return NOT_A_DICT_POS;
+ }
+ return pos + ShortcutListReadingUtils::getShortcutListSizeFieldSize();
+ }
+
+ void getNextShortcut(const int maxCodePointCount, int *const outCodePoint,
+ int *const outCodePointCount, bool *const outIsWhitelist, bool *const outHasNext,
+ int *const pos) const {
+ const bool usesAdditionalBuffer = *pos >= mBufSize;
+ const uint8_t *const buffer = usesAdditionalBuffer
+ ? mAdditionalBuffer->getBuffer() : mShortcutsBuf;
+ if (usesAdditionalBuffer) {
+ *pos -= mBufSize;
+ }
+ const ShortcutListReadingUtils::ShortcutFlags flags =
+ ShortcutListReadingUtils::getFlagsAndForwardPointer(buffer, pos);
+ if (outHasNext) {
+ *outHasNext = ShortcutListReadingUtils::hasNext(flags);
+ }
+ if (outIsWhitelist) {
+ *outIsWhitelist = ShortcutListReadingUtils::isWhitelist(flags);
+ }
+ if (outCodePoint) {
+ *outCodePointCount = ShortcutListReadingUtils::readShortcutTarget(
+ buffer, maxCodePointCount, outCodePoint, pos);
+ }
+ if (usesAdditionalBuffer) {
+ *pos += mBufSize;
+ }
+ }
+
+ void skipAllShortcuts(int *const pos) const {
+ if (*pos >= mBufSize) {
+ *pos -= mBufSize;
+ const int shortcutListSize = ShortcutListReadingUtils
+ ::getShortcutListSizeAndForwardPointer(mAdditionalBuffer->getBuffer(), pos);
+ *pos += mBufSize + shortcutListSize;
+ } else {
+ const int shortcutListSize = ShortcutListReadingUtils
+ ::getShortcutListSizeAndForwardPointer(mShortcutsBuf, pos);
+ *pos += shortcutListSize;
+ }
+ }
+
+ private:
+ DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicShortcutListPolicy);
+
+ const uint8_t *const mShortcutsBuf;
+ const int mBufSize;
+ const ExtendableBuffer *const mAdditionalBuffer;
+};
+} // namespace latinime
+#endif // LATINIME_DYNAMIC_SHORTCUT_LIST_POLICY_H
diff --git a/native/jni/src/suggest/policyimpl/dictionary/shortcut/shortcut_list_reading_utils.h b/native/jni/src/suggest/policyimpl/dictionary/shortcut/shortcut_list_reading_utils.h
index b5bb964d5b..5f4f240f52 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/shortcut/shortcut_list_reading_utils.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/shortcut/shortcut_list_reading_utils.h
@@ -50,6 +50,10 @@ class ShortcutListReadingUtils {
- SHORTCUT_LIST_SIZE_FIELD_SIZE;
}
+ static AK_FORCE_INLINE int getShortcutListSizeFieldSize() {
+ return SHORTCUT_LIST_SIZE_FIELD_SIZE;
+ }
+
static AK_FORCE_INLINE void skipShortcuts(const uint8_t *const dictRoot, int *const pos) {
const int shortcutListSize = getShortcutListSizeAndForwardPointer(dictRoot, pos);
*pos += shortcutListSize;
diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/extendable_buffer.h b/native/jni/src/suggest/policyimpl/dictionary/utils/extendable_buffer.h
index d902d19c8a..5c75027d20 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/utils/extendable_buffer.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/utils/extendable_buffer.h
@@ -30,12 +30,12 @@ class ExtendableBuffer {
public:
ExtendableBuffer() : mBuffer(INITIAL_BUFFER_SIZE), mUsedSize(0) {}
- AK_FORCE_INLINE uint8_t *getBuffer() {
+ AK_FORCE_INLINE const uint8_t *getBuffer() const {
return &mBuffer[0];
}
// Return if the buffer is successfully extended or not.
- AK_FORCE_INLINE bool extendBuffer() {
+ AK_FORCE_INLINE bool extendBuffer() {
if (mBuffer.size() + EXTEND_BUFFER_SIZE_STEP > MAX_BUFFER_SIZE) {
return false;
}
--
GitLab