diff --git a/java/src/com/android/inputmethod/latin/makedict/AbstractDictDecoder.java b/java/src/com/android/inputmethod/latin/makedict/AbstractDictDecoder.java
index e7d1c98a98e3162009a9e548c018434cc24487af..1bc32272407583957dd19c31c1839fc37a3174f6 100644
--- a/java/src/com/android/inputmethod/latin/makedict/AbstractDictDecoder.java
+++ b/java/src/com/android/inputmethod/latin/makedict/AbstractDictDecoder.java
@@ -20,7 +20,6 @@ import com.android.inputmethod.annotations.UsedForTesting;
import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.CharEncoding;
import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.DictBuffer;
import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions;
-import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;
import java.io.FileNotFoundException;
import java.io.IOException;
@@ -116,94 +115,6 @@ public abstract class AbstractDictDecoder implements DictDecoder {
}
}
- /**
- * A utility class for reading a PtNode.
- */
- protected static class PtNodeReader {
- protected static int readPtNodeOptionFlags(final DictBuffer dictBuffer) {
- return dictBuffer.readUnsignedByte();
- }
-
- protected static int readParentAddress(final DictBuffer dictBuffer,
- final FormatOptions formatOptions) {
- if (BinaryDictIOUtils.supportsDynamicUpdate(formatOptions)) {
- return BinaryDictDecoderUtils.readSInt24(dictBuffer);
- } else {
- return FormatSpec.NO_PARENT_ADDRESS;
- }
- }
-
- protected static int readChildrenAddress(final DictBuffer dictBuffer, final int optionFlags,
- final FormatOptions formatOptions) {
- if (BinaryDictIOUtils.supportsDynamicUpdate(formatOptions)) {
- final int address = BinaryDictDecoderUtils.readSInt24(dictBuffer);
- if (address == 0) return FormatSpec.NO_CHILDREN_ADDRESS;
- return address;
- } else {
- switch (optionFlags & FormatSpec.MASK_CHILDREN_ADDRESS_TYPE) {
- case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_ONEBYTE:
- return dictBuffer.readUnsignedByte();
- case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_TWOBYTES:
- return dictBuffer.readUnsignedShort();
- case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_THREEBYTES:
- return dictBuffer.readUnsignedInt24();
- case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_NOADDRESS:
- default:
- return FormatSpec.NO_CHILDREN_ADDRESS;
- }
- }
- }
-
- // Reads shortcuts and returns the read length.
- protected static int readShortcut(final DictBuffer dictBuffer,
- final ArrayList<WeightedString> shortcutTargets) {
- final int pointerBefore = dictBuffer.position();
- dictBuffer.readUnsignedShort(); // skip the size
- while (true) {
- final int targetFlags = dictBuffer.readUnsignedByte();
- final String word = CharEncoding.readString(dictBuffer);
- shortcutTargets.add(new WeightedString(word,
- targetFlags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY));
- if (0 == (targetFlags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT)) break;
- }
- return dictBuffer.position() - pointerBefore;
- }
-
- protected static int readBigramAddresses(final DictBuffer dictBuffer,
- final ArrayList<PendingAttribute> bigrams, final int baseAddress) {
- int readLength = 0;
- int bigramCount = 0;
- while (bigramCount++ < FormatSpec.MAX_BIGRAMS_IN_A_PTNODE) {
- final int bigramFlags = dictBuffer.readUnsignedByte();
- ++readLength;
- final int sign = 0 == (bigramFlags & FormatSpec.FLAG_BIGRAM_ATTR_OFFSET_NEGATIVE)
- ? 1 : -1;
- int bigramAddress = baseAddress + readLength;
- switch (bigramFlags & FormatSpec.MASK_BIGRAM_ATTR_ADDRESS_TYPE) {
- case FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_ONEBYTE:
- bigramAddress += sign * dictBuffer.readUnsignedByte();
- readLength += 1;
- break;
- case FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_TWOBYTES:
- bigramAddress += sign * dictBuffer.readUnsignedShort();
- readLength += 2;
- break;
- case FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_THREEBYTES:
- bigramAddress += sign * dictBuffer.readUnsignedInt24();
- readLength += 3;
- break;
- default:
- throw new RuntimeException("Has bigrams with no address");
- }
- bigrams.add(new PendingAttribute(
- bigramFlags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY,
- bigramAddress));
- if (0 == (bigramFlags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT)) break;
- }
- return readLength;
- }
- }
-
/**
* Check whether the header contains the expected information. This is a no-error method,
* that will return an error code and never throw a checked exception.
@@ -264,9 +175,4 @@ public abstract class AbstractDictDecoder implements DictDecoder {
public boolean hasNextPtNodeArray() {
return false;
}
-
- @Override
- @UsedForTesting
- public void skipPtNode(final FormatOptions formatOptions) {
- }
}
diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java
index 247d81ad34cb0e10703bc249fa00a7b865dccc38..fc5788de95281c4c1dc5f57e58c1982118c8502b 100644
--- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java
+++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java
@@ -166,15 +166,6 @@ public final class BinaryDictDecoderUtils {
return size;
}
- @UsedForTesting
- static int getCharArraySize(final int[] chars, final int start, final int end) {
- int size = 0;
- for (int i = start; i < end; ++i) {
- size += getCharSize(chars[i]);
- }
- return size;
- }
-
/**
* Writes a char array to a byte buffer.
*
@@ -252,41 +243,6 @@ public final class BinaryDictDecoderUtils {
return written;
}
- /**
- * Writes an array of code points with our character format to an OutputStream.
- *
- * This will also write the terminator byte.
- *
- * @param stream the OutputStream to write to.
- * @param codePoints the array of code points
- * @return the size written, in bytes.
- */
- // TODO: Merge this method with writeCharArray and rename the various write* methods to
- // make the difference clear.
- @UsedForTesting
- static int writeCodePoints(final OutputStream stream, final int[] codePoints,
- final int startIndex, final int endIndex)
- throws IOException {
- int written = 0;
- for (int i = startIndex; i < endIndex; ++i) {
- final int codePoint = codePoints[i];
- final int charSize = getCharSize(codePoint);
- if (1 == charSize) {
- stream.write((byte) codePoint);
- } else {
- stream.write((byte) (0xFF & (codePoint >> 16)));
- stream.write((byte) (0xFF & (codePoint >> 8)));
- stream.write((byte) (0xFF & codePoint));
- }
- written += charSize;
- }
- if (endIndex - startIndex > 1) {
- stream.write(FormatSpec.PTNODE_CHARACTERS_TERMINATOR);
- written += FormatSpec.PTNODE_TERMINATOR_SIZE;
- }
- return written;
- }
-
/**
* Reads a string from a DictBuffer. This is the converse of the above method.
*/
@@ -321,50 +277,6 @@ public final class BinaryDictDecoderUtils {
}
}
- // Input methods: Read a binary dictionary to memory.
- // readDictionaryBinary is the public entry point for them.
-
- static int readSInt24(final DictBuffer dictBuffer) {
- final int retval = dictBuffer.readUnsignedInt24();
- final int sign = ((retval & FormatSpec.MSB24) != 0) ? -1 : 1;
- return sign * (retval & FormatSpec.SINT24_MAX);
- }
-
- static int readChildrenAddress(final DictBuffer dictBuffer,
- final int optionFlags, final FormatOptions options) {
- if (options.supportsDynamicUpdate()) {
- final int address = dictBuffer.readUnsignedInt24();
- if (address == 0) return FormatSpec.NO_CHILDREN_ADDRESS;
- if ((address & FormatSpec.MSB24) != 0) {
- return -(address & FormatSpec.SINT24_MAX);
- } else {
- return address;
- }
- }
- switch (optionFlags & FormatSpec.MASK_CHILDREN_ADDRESS_TYPE) {
- case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_ONEBYTE:
- return dictBuffer.readUnsignedByte();
- case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_TWOBYTES:
- return dictBuffer.readUnsignedShort();
- case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_THREEBYTES:
- return dictBuffer.readUnsignedInt24();
- case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_NOADDRESS:
- default:
- return FormatSpec.NO_CHILDREN_ADDRESS;
- }
- }
-
- static int readParentAddress(final DictBuffer dictBuffer,
- final FormatOptions formatOptions) {
- if (BinaryDictIOUtils.supportsDynamicUpdate(formatOptions)) {
- final int parentAddress = dictBuffer.readUnsignedInt24();
- final int sign = ((parentAddress & FormatSpec.MSB24) != 0) ? -1 : 1;
- return sign * (parentAddress & FormatSpec.SINT24_MAX);
- } else {
- return FormatSpec.NO_PARENT_ADDRESS;
- }
- }
-
/**
* Reads and returns the PtNode count out of a buffer and forwards the pointer.
*/
diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java
index 67f86d44e0a7359b04744d449dc9c76423113b82..54446df49031a9ac44adc1e87b0ea731a1138acc 100644
--- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java
+++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java
@@ -21,7 +21,6 @@ import com.android.inputmethod.latin.Constants;
import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.CharEncoding;
import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.DictBuffer;
import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions;
-import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode;
import com.android.inputmethod.latin.utils.ByteArrayDictBuffer;
import java.io.File;
diff --git a/java/src/com/android/inputmethod/latin/makedict/DictDecoder.java b/java/src/com/android/inputmethod/latin/makedict/DictDecoder.java
index bba1d434f9419b23238a54582071300d8047a731..a93a8bbad0861c8fbfb55947e9ef540aeeeb1062 100644
--- a/java/src/com/android/inputmethod/latin/makedict/DictDecoder.java
+++ b/java/src/com/android/inputmethod/latin/makedict/DictDecoder.java
@@ -228,8 +228,6 @@ public interface DictDecoder {
}
}
- public void skipPtNode(final FormatOptions formatOptions);
-
/**
* @return whether this decoder has a valid binary dictionary that it can decode.
*/
diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver2DictDecoder.java b/java/src/com/android/inputmethod/latin/makedict/Ver2DictDecoder.java
index f22b118d941fb63a2cba88ae921380010e60e726..fd6138d41a8d5f69d5ea4313d1cb79c2cd7d4519 100644
--- a/java/src/com/android/inputmethod/latin/makedict/Ver2DictDecoder.java
+++ b/java/src/com/android/inputmethod/latin/makedict/Ver2DictDecoder.java
@@ -20,7 +20,6 @@ import com.android.inputmethod.annotations.UsedForTesting;
import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.CharEncoding;
import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.DictBuffer;
import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions;
-import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode;
import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;
import android.util.Log;
@@ -38,11 +37,82 @@ import java.util.Arrays;
public class Ver2DictDecoder extends AbstractDictDecoder {
private static final String TAG = Ver2DictDecoder.class.getSimpleName();
- protected static class PtNodeReader extends AbstractDictDecoder.PtNodeReader {
+ /**
+ * A utility class for reading a PtNode.
+ */
+ protected static class PtNodeReader {
private static ProbabilityInfo readProbabilityInfo(final DictBuffer dictBuffer) {
// Ver2 dicts don't contain historical information.
return new ProbabilityInfo(dictBuffer.readUnsignedByte());
}
+
+ protected static int readPtNodeOptionFlags(final DictBuffer dictBuffer) {
+ return dictBuffer.readUnsignedByte();
+ }
+
+ protected static int readChildrenAddress(final DictBuffer dictBuffer,
+ final int ptNodeFlags) {
+ switch (ptNodeFlags & FormatSpec.MASK_CHILDREN_ADDRESS_TYPE) {
+ case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_ONEBYTE:
+ return dictBuffer.readUnsignedByte();
+ case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_TWOBYTES:
+ return dictBuffer.readUnsignedShort();
+ case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_THREEBYTES:
+ return dictBuffer.readUnsignedInt24();
+ case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_NOADDRESS:
+ default:
+ return FormatSpec.NO_CHILDREN_ADDRESS;
+ }
+ }
+
+ // Reads shortcuts and returns the read length.
+ protected static int readShortcut(final DictBuffer dictBuffer,
+ final ArrayList<WeightedString> shortcutTargets) {
+ final int pointerBefore = dictBuffer.position();
+ dictBuffer.readUnsignedShort(); // skip the size
+ while (true) {
+ final int targetFlags = dictBuffer.readUnsignedByte();
+ final String word = CharEncoding.readString(dictBuffer);
+ shortcutTargets.add(new WeightedString(word,
+ targetFlags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY));
+ if (0 == (targetFlags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT)) break;
+ }
+ return dictBuffer.position() - pointerBefore;
+ }
+
+ protected static int readBigramAddresses(final DictBuffer dictBuffer,
+ final ArrayList<PendingAttribute> bigrams, final int baseAddress) {
+ int readLength = 0;
+ int bigramCount = 0;
+ while (bigramCount++ < FormatSpec.MAX_BIGRAMS_IN_A_PTNODE) {
+ final int bigramFlags = dictBuffer.readUnsignedByte();
+ ++readLength;
+ final int sign = 0 == (bigramFlags & FormatSpec.FLAG_BIGRAM_ATTR_OFFSET_NEGATIVE)
+ ? 1 : -1;
+ int bigramAddress = baseAddress + readLength;
+ switch (bigramFlags & FormatSpec.MASK_BIGRAM_ATTR_ADDRESS_TYPE) {
+ case FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_ONEBYTE:
+ bigramAddress += sign * dictBuffer.readUnsignedByte();
+ readLength += 1;
+ break;
+ case FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_TWOBYTES:
+ bigramAddress += sign * dictBuffer.readUnsignedShort();
+ readLength += 2;
+ break;
+ case FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_THREEBYTES:
+ bigramAddress += sign * dictBuffer.readUnsignedInt24();
+ readLength += 3;
+ break;
+ default:
+ throw new RuntimeException("Has bigrams with no address");
+ }
+ bigrams.add(new PendingAttribute(
+ bigramFlags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY,
+ bigramAddress));
+ if (0 == (bigramFlags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT)) break;
+ }
+ return readLength;
+ }
}
protected final File mDictionaryBinaryFile;
@@ -96,8 +166,8 @@ public class Ver2DictDecoder extends AbstractDictDecoder {
}
final DictionaryHeader header = super.readHeader(mDictBuffer);
final int version = header.mFormatOptions.mVersion;
- if (!(version >= 2 && version <= 3)) {
- throw new UnsupportedFormatException("File header has a wrong version : " + version);
+ if (version != FormatSpec.VERSION2) {
+ throw new UnsupportedFormatException("File header has a wrong version : " + version);
}
return header;
}
@@ -109,12 +179,6 @@ public class Ver2DictDecoder extends AbstractDictDecoder {
int addressPointer = ptNodePos;
final int flags = PtNodeReader.readPtNodeOptionFlags(mDictBuffer);
addressPointer += FormatSpec.PTNODE_FLAGS_SIZE;
-
- final int parentAddress = PtNodeReader.readParentAddress(mDictBuffer, options);
- if (BinaryDictIOUtils.supportsDynamicUpdate(options)) {
- addressPointer += FormatSpec.PARENT_ADDRESS_SIZE;
- }
-
final int characters[];
if (0 != (flags & FormatSpec.FLAG_HAS_MULTIPLE_CHARS)) {
int index = 0;
@@ -141,7 +205,7 @@ public class Ver2DictDecoder extends AbstractDictDecoder {
} else {
probabilityInfo = null;
}
- int childrenAddress = PtNodeReader.readChildrenAddress(mDictBuffer, flags, options);
+ int childrenAddress = PtNodeReader.readChildrenAddress(mDictBuffer, flags);
if (childrenAddress != FormatSpec.NO_CHILDREN_ADDRESS) {
childrenAddress += addressPointer;
}
@@ -168,7 +232,7 @@ public class Ver2DictDecoder extends AbstractDictDecoder {
bigrams = null;
}
return new PtNodeInfo(ptNodePos, addressPointer, flags, characters, probabilityInfo,
- parentAddress, childrenAddress, shortcutTargets, bigrams);
+ FormatSpec.NO_PARENT_ADDRESS, childrenAddress, shortcutTargets, bigrams);
}
@Override
@@ -224,42 +288,4 @@ public class Ver2DictDecoder extends AbstractDictDecoder {
public boolean hasNextPtNodeArray() {
return mDictBuffer.position() != FormatSpec.NO_FORWARD_LINK_ADDRESS;
}
-
- @Override
- public void skipPtNode(final FormatOptions formatOptions) {
- final int flags = PtNodeReader.readPtNodeOptionFlags(mDictBuffer);
- PtNodeReader.readParentAddress(mDictBuffer, formatOptions);
- BinaryDictIOUtils.skipString(mDictBuffer,
- (flags & FormatSpec.FLAG_HAS_MULTIPLE_CHARS) != 0);
- PtNodeReader.readChildrenAddress(mDictBuffer, flags, formatOptions);
- if ((flags & FormatSpec.FLAG_IS_TERMINAL) != 0) {
- PtNodeReader.readProbabilityInfo(mDictBuffer);
- }
- if ((flags & FormatSpec.FLAG_HAS_SHORTCUT_TARGETS) != 0) {
- final int shortcutsSize = mDictBuffer.readUnsignedShort();
- mDictBuffer.position(mDictBuffer.position() + shortcutsSize
- - FormatSpec.PTNODE_SHORTCUT_LIST_SIZE_SIZE);
- }
- if ((flags & FormatSpec.FLAG_HAS_BIGRAMS) != 0) {
- int bigramCount = 0;
- while (bigramCount++ < FormatSpec.MAX_BIGRAMS_IN_A_PTNODE) {
- final int bigramFlags = mDictBuffer.readUnsignedByte();
- switch (bigramFlags & FormatSpec.MASK_BIGRAM_ATTR_ADDRESS_TYPE) {
- case FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_ONEBYTE:
- mDictBuffer.readUnsignedByte();
- break;
- case FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_TWOBYTES:
- mDictBuffer.readUnsignedShort();
- break;
- case FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_THREEBYTES:
- mDictBuffer.readUnsignedInt24();
- break;
- }
- if ((bigramFlags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT) == 0) break;
- }
- if (bigramCount >= FormatSpec.MAX_BIGRAMS_IN_A_PTNODE) {
- throw new RuntimeException("Too many bigrams in a PtNode.");
- }
- }
- }
}
diff --git a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/XmlDictInputOutput.java b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/XmlDictInputOutput.java
index f45615cbe8ff60657350ce19104cb88817a057d6..2ac842a8046cb6c9e2ab7509251668a90ba68653 100644
--- a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/XmlDictInputOutput.java
+++ b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/XmlDictInputOutput.java
@@ -358,7 +358,6 @@ public class XmlDictInputOutput {
}
// TODO: use an XMLSerializer if this gets big
destination.write("<wordlist format=\"2\"");
- final HashMap<String, String> options = dict.mOptions.mAttributes;
for (final String key : dict.mOptions.mAttributes.keySet()) {
final String value = dict.mOptions.mAttributes.get(key);
destination.write(" " + key + "=\"" + value + "\"");