CSS Tokenizer: Add an on-stack tokenizer scope

Source/core/css/parser/CSSTokenizer.cpp

 // Copyright 2014 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "config.h"
 #include "core/css/parser/CSSTokenizer.h"
 
 namespace blink {
 #include "core/CSSTokenizerCodepoints.cpp"
 }
 
+#include "core/css/parser/CSSParserTokenRange.h"
 #include "core/css/parser/CSSTokenizerInputStream.h"
 #include "core/html/parser/HTMLParserIdioms.h"
 #include "wtf/unicode/CharacterNames.h"
 
 namespace blink {
 
+CSSTokenizer::Scope::Scope(const String& string)
+{
+    // According to the spec, we should perform preprocessing here.
+    // See: http://dev.w3.org/csswg/css-syntax/#input-preprocessing
+    //
+    // However, we can skip this step since:
+    // * We're using HTML spaces (which accept \r and \f as a valid white space)
+    // * Do not count white spaces
+    // * consumeEscape replaces NULLs for replacement characters
+
+    if (string.isEmpty())
+        return;
+
+    // To avoid resizing we err on the side of reserving too much space.
+    // Most strings we tokenize have about 3.5 to 5 characters per token.
+    m_tokens.reserveInitialCapacity(string.length() / 3);
+
+    CSSTokenizerInputStream input(string);
+    CSSTokenizer tokenizer(input);
+    while (true) {
+        CSSParserToken token = tokenizer.nextToken();
+        if (token.type() == EOFToken)
+            return;
+        m_tokens.append(token);
+    }
+}
+
+CSSParserTokenRange CSSTokenizer::Scope::tokenRange()
+{
+    return m_tokens;
+}
+
 // http://dev.w3.org/csswg/css-syntax/#name-start-code-point
 static bool isNameStart(UChar c)
 {
     if (isASCIIAlpha(c))
         return true;
     if (c == '_')
         return true;
     return !isASCII(c);
 }
 
@@ skipping 260 matching lines @@
 CSSParserToken CSSTokenizer::stringStart(UChar cc)
 {
     return consumeStringTokenUntil(cc);
 }
 
 CSSParserToken CSSTokenizer::endOfFile(UChar cc)
 {
     return CSSParserToken(EOFToken);
 }
 
-void CSSTokenizer::tokenize(String string, Vector<CSSParserToken>& outTokens)
-{
-    // According to the spec, we should perform preprocessing here.
-    // See: http://dev.w3.org/csswg/css-syntax/#input-preprocessing
-    //
-    // However, we can skip this step since:
-    // * We're using HTML spaces (which accept \r and \f as a valid white space)
-    // * Do not count white spaces
-    // * consumeEscape replaces NULLs for replacement characters
-
-    if (string.isEmpty())
-        return;
-
-    // To avoid resizing we err on the side of reserving too much space.
-    // Most strings we tokenize have about 3.5 to 5 characters per token.
-    outTokens.reserveInitialCapacity(string.length() / 3);
-
-    CSSTokenizerInputStream input(string);
-    CSSTokenizer tokenizer(input);
-    while (true) {
-        CSSParserToken token = tokenizer.nextToken();
-        if (token.type() == EOFToken)
-            return;
-        outTokens.append(token);
-    }
-}
-
 CSSParserToken CSSTokenizer::nextToken()
 {
     // Unlike the HTMLTokenizer, the CSS Syntax spec is written
     // as a stateless, (fixed-size) look-ahead tokenizer.
     // We could move to the stateful model and instead create
     // states for all the "next 3 codepoints are X" cases.
     // State-machine tokenizers are easier to write to handle
     // incremental tokenization of partial sources.
     // However, for now we follow the spec exactly.
     UChar cc = consume();
@@ skipping 377 matching lines @@
 
 bool CSSTokenizer::nextCharsAreIdentifier()
 {
     UChar first = consume();
     bool areIdentifier = nextCharsAreIdentifier(first);
     reconsume(first);
     return areIdentifier;
 }
 
 } // namespace blink