diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictInputOutput.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictInputOutput.java
index d4f7cab5c6765c1ee0d825c0a3e65ec523d8cdf8..9f7f413319d9e6f2fe566fc3112e58acd3ae6e5b 100644
--- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictInputOutput.java
+++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictInputOutput.java
@@ -34,6 +34,7 @@ import java.util.Arrays;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
+import java.util.Stack;
import java.util.TreeMap;
/**
@@ -197,20 +198,21 @@ public class BinaryDictInputOutput {
public void position(int newPosition);
}
- private static final class ByteBufferWrapper implements FusionDictionaryBufferInterface {
- private ByteBuffer buffer;
- ByteBufferWrapper(final ByteBuffer buffer) {
- this.buffer = buffer;
+ public static final class ByteBufferWrapper implements FusionDictionaryBufferInterface {
+ private ByteBuffer mBuffer;
+
+ public ByteBufferWrapper(final ByteBuffer buffer) {
+ mBuffer = buffer;
}
@Override
public int readUnsignedByte() {
- return ((int)buffer.get()) & 0xFF;
+ return ((int)mBuffer.get()) & 0xFF;
}
@Override
public int readUnsignedShort() {
- return ((int)buffer.getShort()) & 0xFFFF;
+ return ((int)mBuffer.getShort()) & 0xFFFF;
}
@Override
@@ -221,17 +223,17 @@ public class BinaryDictInputOutput {
@Override
public int readInt() {
- return buffer.getInt();
+ return mBuffer.getInt();
}
@Override
public int position() {
- return buffer.position();
+ return mBuffer.position();
}
@Override
public void position(int newPos) {
- buffer.position(newPos);
+ mBuffer.position(newPos);
return;
}
}
@@ -1367,6 +1369,109 @@ public class BinaryDictInputOutput {
return node;
}
+ // TODO: move these methods (readUnigramsAndBigramsBinary(|Inner)) and an inner class (Position)
+ // out of this class.
+ private static class Position {
+ public static final int NOT_READ_GROUPCOUNT = -1;
+
+ public int mAddress;
+ public int mNumOfCharGroup;
+ public int mPosition;
+ public int mLength;
+
+ public Position(int address, int length) {
+ mAddress = address;
+ mLength = length;
+ mNumOfCharGroup = NOT_READ_GROUPCOUNT;
+ }
+ }
+
+ /**
+ * Tours all node without recursive call.
+ */
+ private static void readUnigramsAndBigramsBinaryInner(
+ final FusionDictionaryBufferInterface buffer, final int headerSize,
+ final Map<Integer, String> words, final Map<Integer, Integer> frequencies,
+ final Map<Integer, ArrayList<PendingAttribute>> bigrams) {
+
+ int[] pushedChars = new int[MAX_WORD_LENGTH + 1];
+
+ Stack<Position> stack = new Stack<Position>();
+ int index = 0;
+
+ Position initPos = new Position(headerSize, 0);
+ stack.push(initPos);
+
+ while (!stack.empty()) {
+ Position p = stack.peek();
+
+ if (DBG) {
+ MakedictLog.d("read: address=" + p.mAddress + ", numOfCharGroup=" +
+ p.mNumOfCharGroup + ", position=" + p.mPosition + ", length=" + p.mLength);
+ }
+
+ if (buffer.position() != p.mAddress) buffer.position(p.mAddress);
+ if (index != p.mLength) index = p.mLength;
+
+ if (p.mNumOfCharGroup == Position.NOT_READ_GROUPCOUNT) {
+ p.mNumOfCharGroup = readCharGroupCount(buffer);
+ p.mAddress += getGroupCountSize(p.mNumOfCharGroup);
+ p.mPosition = 0;
+ }
+
+ CharGroupInfo info = readCharGroup(buffer, p.mAddress - headerSize);
+ for (int i = 0; i < info.mCharacters.length; ++i) {
+ pushedChars[index++] = info.mCharacters[i];
+ }
+ p.mPosition++;
+
+ if (info.mFrequency != FusionDictionary.CharGroup.NOT_A_TERMINAL) { // found word
+ words.put(info.mOriginalAddress, new String(pushedChars, 0, index));
+ frequencies.put(info.mOriginalAddress, info.mFrequency);
+ if (info.mBigrams != null) bigrams.put(info.mOriginalAddress, info.mBigrams);
+ }
+
+ if (p.mPosition == p.mNumOfCharGroup) {
+ stack.pop();
+ } else {
+ // the node has more groups.
+ p.mAddress = buffer.position();
+ }
+
+ if (hasChildrenAddress(info.mChildrenAddress)) {
+ Position childrenPos = new Position(info.mChildrenAddress + headerSize, index);
+ stack.push(childrenPos);
+ }
+ }
+
+ return;
+ }
+
+ /**
+ * Reads unigrams and bigrams from the binary file.
+ * Doesn't make the memory representation of the dictionary.
+ *
+ * @param buffer the buffer to read.
+ * @param words the map to store the address as a key and the word as a value.
+ * @param frequencies the map to store the address as a key and the frequency as a value.
+ * @param bigrams the map to store the address as a key and the list of address as a value.
+ * @throws IOException
+ * @throws UnsupportedFormatException
+ */
+ public static void readUnigramsAndBigramsBinary(final FusionDictionaryBufferInterface buffer,
+ final Map<Integer, String> words, final Map<Integer, Integer> frequencies,
+ final Map<Integer, ArrayList<PendingAttribute>> bigrams) throws IOException,
+ UnsupportedFormatException {
+
+ // Read header
+ final int version = checkFormatVersion(buffer);
+ final int optionsFlags = buffer.readUnsignedShort();
+ final HashMap<String, String> options = new HashMap<String, String>();
+ final int headerSize = readHeader(buffer, options, version);
+
+ readUnigramsAndBigramsBinaryInner(buffer, headerSize, words, frequencies, bigrams);
+ }
+
/**
* Helper function to get the binary format version from the header.
* @throws IOException
diff --git a/tests/src/com/android/inputmethod/latin/BinaryDictIOTests.java b/tests/src/com/android/inputmethod/latin/BinaryDictIOTests.java
index 0094db8a7cd6dc38af3e7a8ebd40bdffafbdf058..2cedb5445accaecf149f085ed488da29400bb426 100644
--- a/tests/src/com/android/inputmethod/latin/BinaryDictIOTests.java
+++ b/tests/src/com/android/inputmethod/latin/BinaryDictIOTests.java
@@ -20,6 +20,7 @@ import com.android.inputmethod.latin.makedict.BinaryDictInputOutput;
import com.android.inputmethod.latin.makedict.FusionDictionary;
import com.android.inputmethod.latin.makedict.FusionDictionary.CharGroup;
import com.android.inputmethod.latin.makedict.FusionDictionary.Node;
+import com.android.inputmethod.latin.makedict.PendingAttribute;
import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
import android.test.AndroidTestCase;
@@ -34,7 +35,10 @@ import java.io.FileInputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
import java.util.Random;
import java.util.Set;
@@ -46,6 +50,7 @@ public class BinaryDictIOTests extends AndroidTestCase {
private static final int MAX_UNIGRAMS = 1000;
private static final int UNIGRAM_FREQ = 10;
private static final int BIGRAM_FREQ = 50;
+ private static final int TOLERANCE_OF_BIGRAM_FREQ = 5;
private static final String[] CHARACTERS =
{
@@ -53,6 +58,7 @@ public class BinaryDictIOTests extends AndroidTestCase {
"n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z"
};
+ // Utilities for test
/**
* Generates a random word.
*/
@@ -75,6 +81,9 @@ public class BinaryDictIOTests extends AndroidTestCase {
return new ArrayList<String>(wordSet);
}
+ /**
+ * Adds unigrams to the dictionary.
+ */
private void addUnigrams(final int number,
final FusionDictionary dict,
final List<String> words) {
@@ -86,19 +95,17 @@ public class BinaryDictIOTests extends AndroidTestCase {
private void addBigrams(final FusionDictionary dict,
final List<String> words,
- final SparseArray<List<Integer>> sparseArray) {
- for (int i = 0; i < sparseArray.size(); ++i) {
- final int w1 = sparseArray.keyAt(i);
- for (int w2 : sparseArray.valueAt(i)) {
+ final SparseArray<List<Integer>> bigrams) {
+ for (int i = 0; i < bigrams.size(); ++i) {
+ final int w1 = bigrams.keyAt(i);
+ for (int w2 : bigrams.valueAt(i)) {
dict.setBigram(words.get(w1), words.get(w2), BIGRAM_FREQ);
}
}
}
- private long timeWritingDictToFile(final String fileName,
- final FusionDictionary dict) {
+ private long timeWritingDictToFile(final File file, final FusionDictionary dict) {
- final File file = new File(getContext().getFilesDir(), fileName);
long now = -1, diff = -1;
try {
@@ -140,15 +147,16 @@ public class BinaryDictIOTests extends AndroidTestCase {
}
}
- private long timeReadingAndCheckDict(final String fileName,
- final List<String> words,
+ // Tests for readDictionaryBinary and writeDictionaryBinary
+
+ private long timeReadingAndCheckDict(final File file, final List<String> words,
final SparseArray<List<Integer>> bigrams) {
long now, diff = -1;
+ FileInputStream inStream = null;
try {
- final File file = new File(getContext().getFilesDir(), fileName);
- final FileInputStream inStream = new FileInputStream(file);
+ inStream = new FileInputStream(file);
final ByteBuffer buffer = inStream.getChannel().map(
FileChannel.MapMode.READ_ONLY, 0, file.length());
@@ -166,6 +174,14 @@ public class BinaryDictIOTests extends AndroidTestCase {
Log.e(TAG, "raise IOException while reading file " + e);
} catch (UnsupportedFormatException e) {
Log.e(TAG, "Unsupported format: " + e);
+ } finally {
+ if (inStream != null) {
+ try {
+ inStream.close();
+ } catch (IOException e) {
+ // do nothing
+ }
+ }
}
return diff;
@@ -178,25 +194,26 @@ public class BinaryDictIOTests extends AndroidTestCase {
new FusionDictionary.DictionaryOptions(
new HashMap<String,String>(), false, false));
- final String fileName = generateWord((int)System.currentTimeMillis()) + ".dict";
+ File file = null;
+ try {
+ file = File.createTempFile("runReadAndWrite", ".dict");
+ } catch (IOException e) {
+ Log.e(TAG, "IOException: " + e);
+ }
+
+ assertNotNull(file);
addUnigrams(words.size(), dict, words);
addBigrams(dict, words, bigrams);
// check original dictionary
checkDictionary(dict, words, bigrams);
- final long write = timeWritingDictToFile(fileName, dict);
- final long read = timeReadingAndCheckDict(fileName, words, bigrams);
- deleteFile(fileName);
+ final long write = timeWritingDictToFile(file, dict);
+ final long read = timeReadingAndCheckDict(file, words, bigrams);
return "PROF: read=" + read + "ms, write=" + write + "ms :" + message;
}
- private void deleteFile(final String fileName) {
- final File file = new File(getContext().getFilesDir(), fileName);
- file.delete();
- }
-
public void testReadAndWrite() {
final List<String> results = new ArrayList<String>();
@@ -221,4 +238,134 @@ public class BinaryDictIOTests extends AndroidTestCase {
Log.d(TAG, result);
}
}
+
+ // Tests for readUnigramsAndBigramsBinary
+
+ private void checkWordMap(final List<String> expectedWords,
+ final SparseArray<List<Integer>> expectedBigrams,
+ final Map<Integer, String> resultWords,
+ final Map<Integer, Integer> resultFrequencies,
+ final Map<Integer, ArrayList<PendingAttribute>> resultBigrams) {
+ // check unigrams
+ final Set<String> actualWordsSet = new HashSet<String>(resultWords.values());
+ final Set<String> expectedWordsSet = new HashSet<String>(expectedWords);
+ assertEquals(actualWordsSet, expectedWordsSet);
+
+ for (int freq : resultFrequencies.values()) {
+ assertEquals(freq, UNIGRAM_FREQ);
+ }
+
+ // check bigrams
+ final Map<String, List<String>> expBigrams = new HashMap<String, List<String>>();
+ for (int i = 0; i < expectedBigrams.size(); ++i) {
+ final String word1 = expectedWords.get(expectedBigrams.keyAt(i));
+ for (int w2 : expectedBigrams.valueAt(i)) {
+ if (expBigrams.get(word1) == null) {
+ expBigrams.put(word1, new ArrayList<String>());
+ }
+ expBigrams.get(word1).add(expectedWords.get(w2));
+ }
+ }
+
+ final Map<String, List<String>> actBigrams = new HashMap<String, List<String>>();
+ for (Entry<Integer, ArrayList<PendingAttribute>> entry : resultBigrams.entrySet()) {
+ final String word1 = resultWords.get(entry.getKey());
+ final int unigramFreq = resultFrequencies.get(entry.getKey());
+ for (PendingAttribute attr : entry.getValue()) {
+ final String word2 = resultWords.get(attr.mAddress);
+ if (actBigrams.get(word1) == null) {
+ actBigrams.put(word1, new ArrayList<String>());
+ }
+ actBigrams.get(word1).add(word2);
+
+ final int bigramFreq = BinaryDictInputOutput.reconstructBigramFrequency(
+ unigramFreq, attr.mFrequency);
+ assertTrue(Math.abs(bigramFreq - BIGRAM_FREQ) < TOLERANCE_OF_BIGRAM_FREQ);
+ }
+ }
+
+ assertEquals(actBigrams, expBigrams);
+ }
+
+ private long timeAndCheckReadUnigramsAndBigramsBinary(final File file, final List<String> words,
+ final SparseArray<List<Integer>> bigrams) {
+ FileInputStream inStream = null;
+
+ final Map<Integer, String> resultWords = CollectionUtils.newTreeMap();
+ final Map<Integer, ArrayList<PendingAttribute>> resultBigrams =
+ CollectionUtils.newTreeMap();
+ final Map<Integer, Integer> resultFreqs = CollectionUtils.newTreeMap();
+
+ long now = -1, diff = -1;
+ try {
+ inStream = new FileInputStream(file);
+ final ByteBuffer buffer = inStream.getChannel().map(
+ FileChannel.MapMode.READ_ONLY, 0, file.length());
+
+ now = System.currentTimeMillis();
+ BinaryDictInputOutput.readUnigramsAndBigramsBinary(
+ new BinaryDictInputOutput.ByteBufferWrapper(buffer), resultWords, resultFreqs,
+ resultBigrams);
+ diff = System.currentTimeMillis() - now;
+ checkWordMap(words, bigrams, resultWords, resultFreqs, resultBigrams);
+ } catch (IOException e) {
+ Log.e(TAG, "IOException " + e);
+ } catch (UnsupportedFormatException e) {
+ Log.e(TAG, "UnsupportedFormatException: " + e);
+ } finally {
+ if (inStream != null) {
+ try {
+ inStream.close();
+ } catch (IOException e) {
+ // do nothing
+ }
+ }
+ }
+
+ return diff;
+ }
+
+ private void runReadUnigramsAndBigramsBinary(final List<String> words,
+ final SparseArray<List<Integer>> bigrams) {
+
+ // making the dictionary from lists of words.
+ final FusionDictionary dict = new FusionDictionary(new Node(),
+ new FusionDictionary.DictionaryOptions(
+ new HashMap<String, String>(), false, false));
+
+ File file = null;
+ try {
+ file = File.createTempFile("runReadUnigrams", ".dict");
+ } catch (IOException e) {
+ Log.e(TAG, "IOException: " + e);
+ }
+
+ assertNotNull(file);
+
+ addUnigrams(words.size(), dict, words);
+ addBigrams(dict, words, bigrams);
+ timeWritingDictToFile(file, dict);
+
+ long wordMap = timeAndCheckReadUnigramsAndBigramsBinary(file, words, bigrams);
+ long fullReading = timeReadingAndCheckDict(file, words, bigrams);
+
+ Log.d(TAG, "read=" + fullReading + ", bytearray=" + wordMap);
+ }
+
+ public void testReadUnigramsAndBigramsBinary() {
+ final List<String> results = new ArrayList<String>();
+
+ final Random random = new Random(123456);
+ final List<String> words = generateWords(MAX_UNIGRAMS, random);
+ final SparseArray<List<Integer>> emptyArray = CollectionUtils.newSparseArray();
+
+ runReadUnigramsAndBigramsBinary(words, emptyArray);
+
+ final SparseArray<List<Integer>> star = CollectionUtils.newSparseArray();
+ for (int i = 1; i < words.size(); ++i) {
+ star.put(i-1, new ArrayList<Integer>());
+ star.get(i-1).add(i);
+ }
+ runReadUnigramsAndBigramsBinary(words, star);
+ }
}