Skip to content
Snippets Groups Projects
Commit 5852a259 authored by Tadashi G. Takaoka's avatar Tadashi G. Takaoka
Browse files

Parse escaped sequence strictly in CSV parser 

This change alters the CSV parser behavior. The parser only resolves
  * String resource reference. ["@string/res" -> "<content_of_res>"]
  * Other occurrence of escape sequence will be intact. ["\x" -> "\x"]

Before this change, escape sequence in moreKeys string is parsed three
times. At first in parsing string resource, next in CSV parser, and at
last in KeySpecParser. So that representing single escape character
itself is a bit annoying, "\\\\\\\\".

Now we can represent single escape character itself in string resource by "\\\\".

Change-Id: Ib978e17b779cc82585eed8241ac3857508b14bc7
parent a456c755
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
java/res/values/donottranslate-more-keys.xml
+ 1
− 1
View file @ 5852a259
......@@ -54,7 +54,7 @@
<string name="more_keys_for_currency_euro">¢,£,$,¥,₱</string>
<string name="more_keys_for_currency_pound">¢,$,€,¥,₱</string>
<string name="more_keys_for_currency_general">¢,$,€,£,¥,₱</string>
<string name="more_keys_for_smiley">":-)|:-) ,:-(|:-( ,;-)|;-) ,:-P|:-P ,=-O|=-O ,:-*|:-* ,:O|:O ,B-)|B-) ,:-$|:-$ ,:-!|:-! ,:-[|:-[ ,O:-)|O:-) ,:-\\\\\\\\|:-\\\\\\\\ ,:\'(|:\'( ,:-D|:-D "</string>
<string name="more_keys_for_smiley">":-)|:-) ,:-(|:-( ,;-)|;-) ,:-P|:-P ,=-O|=-O ,:-*|:-* ,:O|:O ,B-)|B-) ,:-$|:-$ ,:-!|:-! ,:-[|:-[ ,O:-)|O:-) ,:-\\\\|:-\\\\ ,:\'(|:\'( ,:-D|:-D "</string>
<string name="more_keys_for_punctuation">"\\,,\?,!,:,-,\',\",(,),/,;,+,&amp;,\@"</string>
<integer name="mini_keyboard_column_for_punctuation">7</integer>
<string name="keyhintlabel_for_punctuation"></string>
......
java/src/com/android/inputmethod/keyboard/internal/KeySpecParser.java
+ 11
− 43
View file @ 5852a259
......@@ -48,7 +48,7 @@ public class KeySpecParser {
private static final char ESCAPE_CHAR = '\\';
private static final char PREFIX_AT = '@';
private static final char SUFFIX_SLASH = '/';
private static final String PREFIX_STRING = PREFIX_AT + "string";
private static final String PREFIX_STRING = PREFIX_AT + "string" + SUFFIX_SLASH;
private static final char LABEL_END = '|';
private static final String PREFIX_ICON = PREFIX_AT + "icon" + SUFFIX_SLASH;
private static final String PREFIX_CODE = PREFIX_AT + "integer" + SUFFIX_SLASH;
......@@ -293,13 +293,11 @@ public class KeySpecParser {
sb.append(res.getString(resId));
pos = end - 1;
} else if (c == ESCAPE_CHAR) {
pos++;
if (sb != null) {
sb.append(c);
if (pos < size) {
sb.append(text.charAt(pos));
}
// Append both escape character and escaped character.
sb.append(text.substring(pos, Math.min(pos + 2, size)));
}
pos++;
} else if (sb != null) {
sb.append(c);
}
......@@ -309,10 +307,7 @@ public class KeySpecParser {
private static int searchResourceNameEnd(String text, int start) {
final int size = text.length();
if (start >= size || text.charAt(start) != SUFFIX_SLASH) {
throw new RuntimeException("Resource name not specified");
}
for (int pos = start + 1; pos < size; pos++) {
for (int pos = start; pos < size; pos++) {
final char c = text.charAt(pos);
// String resource name should be consisted of [a-z_0-9].
if ((c >= 'a' && c <= 'z') || c == '_' || (c >= '0' && c <= '9')) {
......@@ -333,7 +328,6 @@ public class KeySpecParser {
return new String[] { text };
}
final StringBuilder sb = new StringBuilder();
ArrayList<String> list = null;
int start = 0;
for (int pos = 0; pos < size; pos++) {
......@@ -342,44 +336,18 @@ public class KeySpecParser {
if (list == null) {
list = new ArrayList<String>();
}
if (sb.length() == 0) {
list.add(text.substring(start, pos));
} else {
list.add(sb.toString());
sb.setLength(0);
}
list.add(text.substring(start, pos));
// Skip comma
start = pos + 1;
continue;
}
// TODO: Only parse escaped comma. Other escaped character should be passed through
// with escaped character prefixed.
// Skip escaped sequence.
if (c == ESCAPE_CHAR) {
if (start == pos) {
// Skip escaping comma at the beginning of the text.
start++;
pos++;
} else {
if (start < pos && sb.length() == 0) {
sb.append(text.substring(start, pos));
}
// Skip comma
pos++;
if (pos < size) {
sb.append(text.charAt(pos));
}
}
} else if (sb.length() > 0) {
sb.append(c);
} else if (c == ESCAPE_CHAR) {
// Skip escape character and escaped character.
pos++;
}
}
if (list == null) {
return new String[] {
sb.length() > 0 ? sb.toString() : text.substring(start)
};
return new String[] { text.substring(start) };
} else {
list.add(sb.length() > 0 ? sb.toString() : text.substring(start));
list.add(text.substring(start));
return list.toArray(new String[list.size()]);
}
}
......
tests/res/values/donottranslate.xml
+ 3
− 0
View file @ 5852a259
......@@ -30,11 +30,14 @@
<string name="label_surrounded_by_spaces">" abc "</string>
<string name="escaped_char">"\\a"</string>
<string name="escaped_comma">"\\,"</string>
<string name="escaped_comma_escape">"a\\,\\"</string>
<string name="escaped_escape">"\\\\"</string>
<string name="escaped_label">"a\\bc"</string>
<string name="escaped_label_at_beginning">"\\abc"</string>
<string name="escaped_label_at_end">"abc\\"</string>
<string name="escaped_label_with_comma">"a\\,c"</string>
<string name="escaped_label_with_comma_at_beginning">"\\,bc"</string>
<string name="escaped_label_with_comma_at_end">"ab\\,"</string>
<string name="escaped_label_with_successive">"\\,\\\\bc"</string>
<string name="escaped_label_with_escape">"a\\\\c"</string>
<string name="multiple_chars">"a,b,c"</string>
......
tests/src/com/android/inputmethod/keyboard/internal/CsvParserTests.java → tests/src/com/android/inputmethod/keyboard/internal/KeySpecParserCsvTests.java
+ 75
− 47
View file @ 5852a259
......@@ -24,7 +24,7 @@ import com.android.inputmethod.latin.tests.R;
import java.util.Arrays;
public class CsvParserTests extends AndroidTestCase {
public class KeySpecParserCsvTests extends AndroidTestCase {
private Resources mTestResources;
@Override
......@@ -79,6 +79,7 @@ public class CsvParserTests extends AndroidTestCase {
public void testParseCsvTextSingle() {
assertTextArray("Single char", "a", "a");
assertTextArray("Surrogate pair", PAIR1, PAIR1);
assertTextArray("Single escape", "\\", "\\");
assertTextArray("Space", " ", " ");
assertTextArray("Single label", "abc", "abc");
assertTextArray("Single srrogate pairs label", SURROGATE2, SURROGATE2);
......@@ -98,36 +99,49 @@ public class CsvParserTests extends AndroidTestCase {
"ab" + SURROGATE1 + "cd");
assertTextArray("Incomplete resource reference 1", "string", "string");
assertTextArray("Incomplete resource reference 2", "@strin", "@strin");
assertTextArray("Incomplete resource reference 3", "@" + SURROGATE2, "@" + SURROGATE2);
assertTextArray("Incomplete resource reference 2", "@string", "@string");
assertTextArray("Incomplete resource reference 3", "string/", "string/");
assertTextArray("Incomplete resource reference 4", "@" + SURROGATE2, "@" + SURROGATE2);
}
public void testParseCsvTextSingleEscaped() {
assertTextArray("Escaped char", "\\a", "a");
assertTextArray("Escaped surrogate pair", "\\" + PAIR1, PAIR1);
assertTextArray("Escaped comma", "\\,", ",");
assertTextArray("Escaped escape", "\\\\", "\\");
assertTextArray("Escaped label", "a\\bc", "abc");
assertTextArray("Escaped surrogate", "a\\" + PAIR1 + "c", "a" + PAIR1 + "c");
assertTextArray("Escaped label at beginning", "\\abc", "abc");
assertTextArray("Escaped surrogate at beginning", "\\" + SURROGATE2, SURROGATE2);
assertTextArray("Escaped label with comma", "a\\,c", "a,c");
assertTextArray("Escaped surrogate with comma", PAIR1 + "\\," + PAIR2, PAIR1 + "," + PAIR2);
assertTextArray("Escaped label with comma at beginning", "\\,bc", ",bc");
assertTextArray("Escaped char", "\\a", "\\a");
assertTextArray("Escaped surrogate pair", "\\" + PAIR1, "\\" + PAIR1);
assertTextArray("Escaped comma", "\\,", "\\,");
assertTextArray("Escaped comma escape", "a\\,\\", "a\\,\\");
assertTextArray("Escaped escape", "\\\\", "\\\\");
assertTextArray("Escaped label", "a\\bc", "a\\bc");
assertTextArray("Escaped surrogate", "a\\" + PAIR1 + "c", "a\\" + PAIR1 + "c");
assertTextArray("Escaped label at beginning", "\\abc", "\\abc");
assertTextArray("Escaped surrogate at beginning", "\\" + SURROGATE2, "\\" + SURROGATE2);
assertTextArray("Escaped label at end", "abc\\", "abc\\");
assertTextArray("Escaped surrogate at end", SURROGATE2 + "\\", SURROGATE2 + "\\");
assertTextArray("Escaped label with comma", "a\\,c", "a\\,c");
assertTextArray("Escaped surrogate with comma",
PAIR1 + "\\," + PAIR2, PAIR1 + "\\," + PAIR2);
assertTextArray("Escaped label with comma at beginning", "\\,bc", "\\,bc");
assertTextArray("Escaped surrogate with comma at beginning",
"\\," + SURROGATE1, "," + SURROGATE1);
assertTextArray("Escaped label with successive", "\\,\\\\bc", ",\\bc");
"\\," + SURROGATE1, "\\," + SURROGATE1);
assertTextArray("Escaped label with comma at end", "ab\\,", "ab\\,");
assertTextArray("Escaped surrogate with comma at end",
SURROGATE2 + "\\,", SURROGATE2 + "\\,");
assertTextArray("Escaped label with successive", "\\,\\\\bc", "\\,\\\\bc");
assertTextArray("Escaped surrogate with successive",
"\\,\\\\" + SURROGATE1, ",\\" + SURROGATE1);
assertTextArray("Escaped label with escape", "a\\\\c", "a\\c");
"\\,\\\\" + SURROGATE1, "\\,\\\\" + SURROGATE1);
assertTextArray("Escaped label with escape", "a\\\\c", "a\\\\c");
assertTextArray("Escaped surrogate with escape",
PAIR1 + "\\\\" + PAIR2, PAIR1 + "\\" + PAIR2);
PAIR1 + "\\\\" + PAIR2, PAIR1 + "\\\\" + PAIR2);
assertTextArray("Escaped @string", "\\@string/empty_string", "@string/empty_string");
assertTextArray("Escaped @string", "\\@string", "\\@string");
assertTextArray("Escaped @string/", "\\@string/", "\\@string/");
assertTextArray("Escaped @string/", "\\@string/empty_string", "\\@string/empty_string");
}
public void testParseCsvTextMulti() {
assertTextArray("Multiple chars", "a,b,c", "a", "b", "c");
assertTextArray("Multiple chars", "a,b,\\c", "a", "b", "\\c");
assertTextArray("Multiple chars and escape at beginning and end",
"\\a,b,\\c\\", "\\a", "b", "\\c\\");
assertTextArray("Multiple surrogates", PAIR1 + "," + PAIR2 + "," + PAIR3,
PAIR1, PAIR2, PAIR3);
assertTextArray("Multiple chars surrounded by spaces", " a , b , c ", " a ", " b ", " c ");
......@@ -139,24 +153,24 @@ public class CsvParserTests extends AndroidTestCase {
}
public void testParseCsvTextMultiEscaped() {
assertTextArray("Multiple chars with comma", "a,\\,,c", "a", ",", "c");
assertTextArray("Multiple chars with comma", "a,\\,,c", "a", "\\,", "c");
assertTextArray("Multiple chars with comma surrounded by spaces", " a , \\, , c ",
" a ", " , ", " c ");
assertTextArray("Multiple labels with escape", "\\abc,d\\ef,gh\\i", "abc", "def", "ghi");
" a ", " \\, ", " c ");
assertTextArray("Multiple labels with escape",
"\\abc,d\\ef,gh\\i", "\\abc", "d\\ef", "gh\\i");
assertTextArray("Multiple labels with escape surrounded by spaces",
" \\abc , d\\ef , gh\\i ", " abc ", " def ", " ghi ");
" \\abc , d\\ef , gh\\i ", " \\abc ", " d\\ef ", " gh\\i ");
assertTextArray("Multiple labels with comma and escape",
"ab\\\\,d\\\\\\,,g\\,i", "ab\\", "d\\,", "g,i");
"ab\\\\,d\\\\\\,,g\\,i", "ab\\\\", "d\\\\\\,", "g\\,i");
assertTextArray("Multiple labels with comma and escape surrounded by spaces",
" ab\\\\ , d\\\\\\, , g\\,i ", " ab\\ ", " d\\, ", " g,i ");
" ab\\\\ , d\\\\\\, , g\\,i ", " ab\\\\ ", " d\\\\\\, ", " g\\,i ");
assertTextArray("Multiple escaped @string", "\\@,\\@string/empty_string",
"@", "@string/empty_string");
"\\@", "\\@string/empty_string");
}
public void testParseCsvResourceError() {
assertError("Incomplete resource name 1", "@string", "@string");
assertError("Incomplete resource name 2", "@string/", "@string/");
assertError("Incomplete resource name", "@string/", "@string/");
assertError("Non existing resource", "@string/non_existing");
}
......@@ -182,27 +196,36 @@ public class CsvParserTests extends AndroidTestCase {
"@string/spaces_at_end_of_label", "abc ");
assertTextArray("label surrounded by spaces",
"@string/label_surrounded_by_spaces", " abc ");
assertTextArray("Escape and single char",
"\\\\@string/single_char", "\\\\a");
}
public void testParseCsvResourceSingleEscaped() {
assertTextArray("Escaped char",
"@string/escaped_char", "a");
"@string/escaped_char", "\\a");
assertTextArray("Escaped comma",
"@string/escaped_comma", ",");
"@string/escaped_comma", "\\,");
assertTextArray("Escaped comma escape",
"@string/escaped_comma_escape", "a\\,\\");
assertTextArray("Escaped escape",
"@string/escaped_escape", "\\");
"@string/escaped_escape", "\\\\");
assertTextArray("Escaped label",
"@string/escaped_label", "abc");
"@string/escaped_label", "a\\bc");
assertTextArray("Escaped label at beginning",
"@string/escaped_label_at_beginning", "abc");
"@string/escaped_label_at_beginning", "\\abc");
assertTextArray("Escaped label at end",
"@string/escaped_label_at_end", "abc\\");
assertTextArray("Escaped label with comma",
"@string/escaped_label_with_comma", "a,c");
"@string/escaped_label_with_comma", "a\\,c");
assertTextArray("Escaped label with comma at beginning",
"@string/escaped_label_with_comma_at_beginning", ",bc");
"@string/escaped_label_with_comma_at_beginning", "\\,bc");
assertTextArray("Escaped label with comma at end",
"@string/escaped_label_with_comma_at_end", "ab\\,");
assertTextArray("Escaped label with successive",
"@string/escaped_label_with_successive", ",\\bc");
"@string/escaped_label_with_successive", "\\,\\\\bc");
assertTextArray("Escaped label with escape",
"@string/escaped_label_with_escape", "a\\c");
"@string/escaped_label_with_escape", "a\\\\c");
}
public void testParseCsvResourceMulti() {
......@@ -220,36 +243,41 @@ public class CsvParserTests extends AndroidTestCase {
public void testParseCsvResourcetMultiEscaped() {
assertTextArray("Multiple chars with comma",
"@string/multiple_chars_with_comma",
"a", ",", "c");
"a", "\\,", "c");
assertTextArray("Multiple chars with comma surrounded by spaces",
"@string/multiple_chars_with_comma_surrounded_by_spaces",
" a ", " , ", " c ");
" a ", " \\, ", " c ");
assertTextArray("Multiple labels with escape",
"@string/multiple_labels_with_escape",
"abc", "def", "ghi");
"\\abc", "d\\ef", "gh\\i");
assertTextArray("Multiple labels with escape surrounded by spaces",
"@string/multiple_labels_with_escape_surrounded_by_spaces",
" abc ", " def ", " ghi ");
" \\abc ", " d\\ef ", " gh\\i ");
assertTextArray("Multiple labels with comma and escape",
"@string/multiple_labels_with_comma_and_escape",
"ab\\", "d\\,", "g,i");
"ab\\\\", "d\\\\\\,", "g\\,i");
assertTextArray("Multiple labels with comma and escape surrounded by spaces",
"@string/multiple_labels_with_comma_and_escape_surrounded_by_spaces",
" ab\\ ", " d\\, ", " g,i ");
" ab\\\\ ", " d\\\\\\, ", " g\\,i ");
}
public void testParseMultipleResources() {
assertTextArray("Literals and resources",
"1,@string/multiple_chars,z", "1", "a", "b", "c", "z");
assertTextArray("Literals and resources and escape at end",
"\\1,@string/multiple_chars,z\\", "\\1", "a", "b", "c", "z\\");
assertTextArray("Multiple single resource chars and labels",
"@string/single_char,@string/single_label,@string/escaped_comma",
"a", "abc", ",");
"a", "abc", "\\,");
assertTextArray("Multiple single resource chars and labels 2",
"@string/single_char,@string/single_label,@string/escaped_comma_escape",
"a", "abc", "a\\,\\");
assertTextArray("Multiple multiple resource chars and labels",
"@string/multiple_chars,@string/multiple_labels,@string/multiple_chars_with_comma",
"a", "b", "c", "abc", "def", "ghi", "a", ",", "c");
"a", "b", "c", "abc", "def", "ghi", "a", "\\,", "c");
assertTextArray("Concatenated resources",
"@string/multiple_chars@string/multiple_labels@string/multiple_chars_with_comma",
"a", "b", "cabc", "def", "ghia", ",", "c");
"a", "b", "cabc", "def", "ghia", "\\,", "c");
assertTextArray("Concatenated resource and literal",
"abc@string/multiple_labels",
"abcabc", "def", "ghi");
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment