Bug 241788 - mozilla::Tokenizer - token type for \n should whitespace if given in constructor r=honzab
--- a/xpcom/ds/Tokenizer.cpp
+++ b/xpcom/ds/Tokenizer.cpp
@@ -255,22 +255,22 @@ Tokenizer::Parse(Token& aToken) const
PARSE_WS,
PARSE_CHAR,
} state;
if (IsWordFirst(*next)) {
state = PARSE_WORD;
} else if (IsNumber(*next)) {
state = PARSE_INTEGER;
+ } else if (strchr(mWhitespaces, *next)) { // not UTF-8 friendly?
+ state = PARSE_WS;
} else if (*next == '\r') {
state = PARSE_CRLF;
} else if (*next == '\n') {
state = PARSE_LF;
- } else if (strchr(mWhitespaces, *next)) { // not UTF-8 friendly?
- state = PARSE_WS;
} else {
state = PARSE_CHAR;
}
mozilla::CheckedUint64 resultingNumber = 0;
while (next < mEnd) {
switch (state) {
--- a/xpcom/tests/gtest/TestTokenizer.cpp
+++ b/xpcom/tests/gtest/TestTokenizer.cpp
@@ -521,16 +521,31 @@ TEST(Tokenizer, SkipWhites)
EXPECT_TRUE(p.CheckEOL());
p.SkipWhites();
EXPECT_TRUE(p.CheckWord("Text4"));
p.SkipWhites(Tokenizer::INCLUDE_NEW_LINE);
EXPECT_TRUE(p.CheckEOF());
}
+TEST(Tokenizer, SkipCustomWhites)
+{
+ Tokenizer p("Text1 \n\r\t.Text2 \n\r\t.", " \n\r\t.");
+
+ EXPECT_TRUE(p.CheckWord("Text1"));
+ p.SkipWhites();
+ EXPECT_TRUE(p.CheckWord("Text2"));
+ EXPECT_TRUE(p.CheckWhite());
+ EXPECT_TRUE(p.CheckWhite());
+ EXPECT_TRUE(p.CheckWhite());
+ EXPECT_TRUE(p.CheckWhite());
+ EXPECT_TRUE(p.CheckWhite());
+ EXPECT_TRUE(p.CheckEOF());
+}
+
TEST(Tokenizer, IntegerReading)
{
#define INT_6_BITS 64U
#define INT_30_BITS 1073741824UL
#define INT_32_BITS 4294967295UL
#define INT_50_BITS 1125899906842624ULL
#define STR_INT_MORE_THAN_64_BITS "922337203685477580899"