Template parser

utils/css/tokenizer.dart

 // Copyright (c) 2011, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
-class Tokenizer extends lang.TokenizerBase {
+class Tokenizer extends CSSTokenizerBase {
   TokenKind cssTokens;
 
   bool _selectorParsing;
 
-  Tokenizer(lang.SourceFile source, bool skipWhitespace, [int index = 0])
+  Tokenizer(SourceFile source, bool skipWhitespace, [int index = 0])
     : super(source, skipWhitespace, index), _selectorParsing = false {
     cssTokens = new TokenKind();
   }
 
-  lang.Token next() {
+  int get startIndex() => _startIndex;
+
+  Token next() {
     // keep track of our starting position
     _startIndex = _index;
 
     if (_interpStack != null && _interpStack.depth == 0) {
       var istack = _interpStack;
       _interpStack = _interpStack.pop();
 
       /* TODO(terry): Enable for variable and string interpolation.
        * if (istack.isMultiline) {
        *   return finishMultilineStringBody(istack.quote);
@@ skipping 14 matching lines @@
       case cssTokens.tokens[TokenKind.RETURN]:
         return finishWhitespace();
       case cssTokens.tokens[TokenKind.END_OF_FILE]:
         return _finishToken(TokenKind.END_OF_FILE);
       case cssTokens.tokens[TokenKind.AT]:
         return _finishToken(TokenKind.AT);
       case cssTokens.tokens[TokenKind.DOT]:
         int start = _startIndex;             // Start where the dot started.
         if (maybeEatDigit()) {
           // looks like a number dot followed by digit(s).
-          lang.Token num = finishNumber();
-          if (num.kind == TokenKind.INTEGER) {
+          Token number = finishNumber();
+          if (number.kind == TokenKind.INTEGER) {
             // It's a number but it's preceeded by a dot, so make it a double.
             _startIndex = start;
             return _finishToken(TokenKind.DOUBLE);
           } else {
             // Don't allow dot followed by a double (e.g,  '..1').
             return _errorToken();
           }
         } else {
           // It's really a dot.
           return _finishToken(TokenKind.DOT);
@@ skipping 15 matching lines @@
       case cssTokens.tokens[TokenKind.PLUS]:
         if (maybeEatDigit()) {
           return finishNumber();
         } else {
           return _finishToken(TokenKind.PLUS);
         }
       case cssTokens.tokens[TokenKind.MINUS]:
         if (maybeEatDigit()) {
           return finishNumber();
         } else if (TokenizerHelpers.isIdentifierStart(ch)) {
-          return this.finishIdentifier();
+          return this.finishIdentifier(ch);
         } else {
           return _finishToken(TokenKind.MINUS);
         }
       case cssTokens.tokens[TokenKind.GREATER]:
         return _finishToken(TokenKind.GREATER);
       case cssTokens.tokens[TokenKind.TILDE]:
         if (_maybeEatChar(cssTokens.tokens[TokenKind.EQUALS])) {
           return _finishToken(TokenKind.INCLUDES);          // ~=
         } else {
           return _finishToken(TokenKind.TILDE);
@@ skipping 46 matching lines @@
         } else {
           return _finishToken(TokenKind.CARET);
         }
       case cssTokens.tokens[TokenKind.DOLLAR]:
         if (_maybeEatChar(cssTokens.tokens[TokenKind.EQUALS])) {
           return _finishToken(TokenKind.SUFFIX_MATCH);    // $=
         } else {
           return _finishToken(TokenKind.DOLLAR);
         }
       case cssTokens.tokens[TokenKind.BANG]:
-        lang.Token tok = finishIdentifier();
+        Token tok = finishIdentifier(ch);
         return (tok == null) ? _finishToken(TokenKind.BANG) : tok;
       default:
         if (TokenizerHelpers.isIdentifierStart(ch)) {
-          return this.finishIdentifier();
+          return this.finishIdentifier(ch);
         } else if (isDigit(ch)) {
           return this.finishNumber();
         } else {
           return _errorToken();
         }
     }
   }
 
   // TODO(jmesserly): we need a way to emit human readable error messages from
   // the tokenizer.
-  lang.Token _errorToken() {
+  Token _errorToken([String message = null]) {
     return _finishToken(TokenKind.ERROR);
   }
 
   int getIdentifierKind() {
     // Is the identifier a unit type?
     int tokId = TokenKind.matchUnits(_text, _startIndex, _index - _startIndex);
     if (tokId == -1) {
       // No, is it a directive?
       tokId = TokenKind.matchDirectives(_text, _startIndex, _index - _startIndex);
     }
     if (tokId == -1) {
       tokId = (_text.substring(_startIndex, _index) == '!important') ?
           TokenKind.IMPORTANT : -1;
     }
 
     return tokId >= 0 ? tokId : TokenKind.IDENTIFIER;
   }
 
   // Need to override so CSS version of isIdentifierPart is used.
-  lang.Token finishIdentifier() {
+  Token finishIdentifier(int ch) {
     while (_index < _text.length) {
 //      if (!TokenizerHelpers.isIdentifierPart(_text.charCodeAt(_index++))) {
       if (!TokenizerHelpers.isIdentifierPart(_text.charCodeAt(_index))) {
 //        _index--;
         break;
       } else {
         _index += 1;
       }
     }
     if (_interpStack != null && _interpStack.depth == -1) {
       _interpStack.depth = 0;
     }
     int kind = getIdentifierKind();
     if (kind == TokenKind.IDENTIFIER) {
       return _finishToken(TokenKind.IDENTIFIER);
     } else {
       return _finishToken(kind);
     }
   }
 
-  lang.Token finishImportant() {
+  Token finishImportant() {
     
   }
 
-  lang.Token finishNumber() {
+  Token finishNumber() {
     eatDigits();
 
     if (_peekChar() == 46/*.*/) {
       // Handle the case of 1.toString().
       _nextChar();
       if (isDigit(_peekChar())) {
         eatDigits();
         return _finishToken(TokenKind.DOUBLE);
       } else {
         _index -= 1;
@@ skipping 22 matching lines @@
   }
 
   bool maybeEatHexDigit() {
     if (_index < _text.length && isHexDigit(_text.charCodeAt(_index))) {
       _index += 1;
       return true;
     }
     return false;
   }
 
-  lang.Token finishMultiLineComment() {
+  Token finishMultiLineComment() {
     while (true) {
       int ch = _nextChar();
       if (ch == 0) {
         return _finishToken(TokenKind.INCOMPLETE_COMMENT);
       } else if (ch == 42/*'*'*/) {
         if (_maybeEatChar(47/*'/'*/)) {
           if (_skipWhitespace) {
             return next();
           } else {
             return _finishToken(TokenKind.COMMENT);
@@ skipping 11 matching lines @@
           }
         }
       }
     }
     return _errorToken();
   }
 
 }
 
 /** Static helper methods. */
+/** Static helper methods. */
 class TokenizerHelpers {
-  static bool isIdentifierStart(int c) =>
-      lang.TokenizerHelpers.isIdentifierStart(c) || c == 95 /*_*/ ||
-      c == 45; /*-*/
+  
+  static bool isIdentifierStart(int c) {
+    return ((c >= 97/*a*/ && c <= 122/*z*/) || (c >= 65/*A*/ && c <= 90/*Z*/) ||
+        c == 95/*_*/ || c == 45 /*-*/);
+  }
 
-  static bool isDigit(int c) => lang.TokenizerHelpers.isDigit(c);
+  static bool isDigit(int c) {
+    return (c >= 48/*0*/ && c <= 57/*9*/);
+  }
 
-  static bool isHexDigit(int c) => lang.TokenizerHelpers.isHexDigit(c);
+  static bool isHexDigit(int c) {
+    return (isDigit(c) || (c >= 97/*a*/ && c <= 102/*f*/) || (c >= 65/*A*/ && c <= 70/*F*/));
+  }
 
-  static bool isWhitespace(int c) => lang.TokenizerHelpers.isWhitespace(c);
+  static bool isWhitespace(int c) {
+    return (c == 32/*' '*/ || c == 9/*'\t'*/ || c == 10/*'\n'*/ || c == 13/*'\r'*/);
+  }
 
-  static bool isIdentifierPart(int c) =>
-      lang.TokenizerHelpers.isIdentifierPart(c) || c == 45 /*-*/;
+  static bool isIdentifierPart(int c) {
+    return (isIdentifierStart(c) || isDigit(c) || c == 45 /*-*/);
+  }
+
+  static bool isInterpIdentifierPart(int c) {
+    return (isIdentifierStart(c) || isDigit(c));
+  }
 }
+