CSS Tokenizer: Add an on-stack tokenizer scope

Source/core/css/parser/CSSTokenizerTest.cpp

 // Copyright 2014 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "config.h"
 #include "core/css/parser/CSSTokenizer.h"
 
+#include "core/css/parser/CSSParserTokenRange.h"
 #include "core/css/parser/MediaQueryBlockWatcher.h"
 #include <gtest/gtest.h>
 
 namespace blink {
 
 // This let's us see the line numbers of failing tests
 #define TEST_TOKENS(string, ...) { \
     String s = string; \
     SCOPED_TRACE(s.ascii().data()); \
     testTokens(string, __VA_ARGS__); \
@@ skipping 40 matching lines @@
 void testTokens(const String& string, const CSSParserToken& token1, const CSSParserToken& token2 = CSSParserToken(EOFToken), const CSSParserToken& token3 = CSSParserToken(EOFToken))
 {
     Vector<CSSParserToken> expectedTokens;
     expectedTokens.append(token1);
     if (token2.type() != EOFToken) {
         expectedTokens.append(token2);
         if (token3.type() != EOFToken)
             expectedTokens.append(token3);
     }
 
-    Vector<CSSParserToken> actualTokens;
-    CSSTokenizer::tokenize(string, actualTokens);
+    CSSParserTokenRange expected(expectedTokens);
 
-    ASSERT_EQ(expectedTokens.size(), actualTokens.size());
-    for (size_t i = 0; i < expectedTokens.size(); ++i)
-        compareTokens(expectedTokens[i], actualTokens[i]);
+    CSSTokenizer::Scope actualScope(string);
+    CSSParserTokenRange actual = actualScope.tokenRange();
+
+    while (!expected.atEnd() || !actual.atEnd())
+        compareTokens(expected.consume(), actual.consume());
 }
 
 static CSSParserToken ident(const String& string) { return CSSParserToken(IdentToken, string); }
 static CSSParserToken atKeyword(const String& string) { return CSSParserToken(AtKeywordToken, string); }
 static CSSParserToken string(const String& string) { return CSSParserToken(StringToken, string); }
 static CSSParserToken function(const String& string) { return CSSParserToken(FunctionToken, string); }
 static CSSParserToken url(const String& string) { return CSSParserToken(UrlToken, string); }
 static CSSParserToken hash(const String& string, HashTokenType type) { return CSSParserToken(type, string); }
 static CSSParserToken delim(char c) { return CSSParserToken(DelimiterToken, c); }
 static CSSParserToken unicodeRange(UChar32 start, UChar32 end) { return CSSParserToken(UnicodeRangeToken, start, end); }
@@ skipping 364 matching lines @@
         {"((), (max-width: 900px)", 2, 1},
         {"(foo(), (max-width: 900px)", 2, 1},
         {"[](()), (max-width: 900px)", 2, 0},
         {"all an[isdfs bla())(i())]icalc(i)(()), (max-width: 400px)", 3, 0},
         {"all an[isdfs bla())(]icalc(i)(()), (max-width: 500px)", 4, 2},
         {"all an[isdfs bla())(]icalc(i)(())), (max-width: 600px)", 4, 1},
         {"all an[isdfs bla())(]icalc(i)(()))], (max-width: 800px)", 4, 0},
         {0, 0, 0} // Do not remove the terminator line.
     };
     for (int i = 0; testCases[i].input; ++i) {
-        Vector<CSSParserToken> tokens;
-        CSSTokenizer::tokenize(testCases[i].input, tokens);
+        CSSTokenizer::Scope scope(testCases[i].input);
+        CSSParserTokenRange range = scope.tokenRange();
         MediaQueryBlockWatcher blockWatcher;
 
         unsigned maxLevel = 0;
         unsigned level = 0;
-        for (size_t j = 0; j < tokens.size(); ++j) {
-            blockWatcher.handleToken(tokens[j]);
+        while (!range.atEnd()) {
+            blockWatcher.handleToken(range.consume());
             level = blockWatcher.blockLevel();
             maxLevel = std::max(level, maxLevel);
         }
         ASSERT_EQ(testCases[i].maxLevel, maxLevel);
         ASSERT_EQ(testCases[i].finalLevel, level);
     }
 }
 
 } // namespace