Bug 241788 - mozilla::Tokenizer - token type for \n should whitespace if given in constructor r=honzab draft
authorValentin Gosu <valentin.gosu@gmail.com>
Thu, 28 Jan 2016 21:32:45 +0100
changeset 327081 b998e916a040cd36ef64161747c7cc1e5ef87d1d
parent 324607 35dd37ee132bc2b85bdae2f83eabc967a631252b
child 327082 776002cfda394c1f953e64f9da22ac4644be3282
push id10188
push uservalentin.gosu@gmail.com
push dateFri, 29 Jan 2016 15:26:10 +0000
reviewershonzab
bugs241788
milestone46.0a1
Bug 241788 - mozilla::Tokenizer - token type for \n should whitespace if given in constructor r=honzab
xpcom/ds/Tokenizer.cpp
xpcom/tests/gtest/TestTokenizer.cpp
--- a/xpcom/ds/Tokenizer.cpp
+++ b/xpcom/ds/Tokenizer.cpp
@@ -255,22 +255,22 @@ Tokenizer::Parse(Token& aToken) const
     PARSE_WS,
     PARSE_CHAR,
   } state;
 
   if (IsWordFirst(*next)) {
     state = PARSE_WORD;
   } else if (IsNumber(*next)) {
     state = PARSE_INTEGER;
+  } else if (strchr(mWhitespaces, *next)) { // not UTF-8 friendly?
+    state = PARSE_WS;
   } else if (*next == '\r') {
     state = PARSE_CRLF;
   } else if (*next == '\n') {
     state = PARSE_LF;
-  } else if (strchr(mWhitespaces, *next)) { // not UTF-8 friendly?
-    state = PARSE_WS;
   } else {
     state = PARSE_CHAR;
   }
 
   mozilla::CheckedUint64 resultingNumber = 0;
 
   while (next < mEnd) {
     switch (state) {
--- a/xpcom/tests/gtest/TestTokenizer.cpp
+++ b/xpcom/tests/gtest/TestTokenizer.cpp
@@ -521,16 +521,31 @@ TEST(Tokenizer, SkipWhites)
   EXPECT_TRUE(p.CheckEOL());
   p.SkipWhites();
 
   EXPECT_TRUE(p.CheckWord("Text4"));
   p.SkipWhites(Tokenizer::INCLUDE_NEW_LINE);
   EXPECT_TRUE(p.CheckEOF());
 }
 
+TEST(Tokenizer, SkipCustomWhites)
+{
+  Tokenizer p("Text1 \n\r\t.Text2 \n\r\t.", " \n\r\t.");
+
+  EXPECT_TRUE(p.CheckWord("Text1"));
+  p.SkipWhites();
+  EXPECT_TRUE(p.CheckWord("Text2"));
+  EXPECT_TRUE(p.CheckWhite());
+  EXPECT_TRUE(p.CheckWhite());
+  EXPECT_TRUE(p.CheckWhite());
+  EXPECT_TRUE(p.CheckWhite());
+  EXPECT_TRUE(p.CheckWhite());
+  EXPECT_TRUE(p.CheckEOF());
+}
+
 TEST(Tokenizer, IntegerReading)
 {
 #define INT_6_BITS                 64U
 #define INT_30_BITS                1073741824UL
 #define INT_32_BITS                4294967295UL
 #define INT_50_BITS                1125899906842624ULL
 #define STR_INT_MORE_THAN_64_BITS "922337203685477580899"