| OLD | NEW |
| 1 // Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 class Tokenizer extends TokenizerBase { | 5 class Tokenizer extends TokenizerBase { |
| 6 TokenKind tmplTokens; | 6 TokenKind tmplTokens; |
| 7 | 7 |
| 8 bool _selectorParsing; | 8 bool _selectorParsing; |
| 9 | 9 |
| 10 Tokenizer(SourceFile source, bool skipWhitespace, [int index = 0]) | 10 Tokenizer(SourceFile source, bool skipWhitespace, [int index = 0]) |
| (...skipping 54 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 65 case tmplTokens.tokens[TokenKind.GREATER_THAN]: | 65 case tmplTokens.tokens[TokenKind.GREATER_THAN]: |
| 66 return _finishToken(TokenKind.GREATER_THAN); | 66 return _finishToken(TokenKind.GREATER_THAN); |
| 67 case tmplTokens.tokens[TokenKind.EQUAL]: | 67 case tmplTokens.tokens[TokenKind.EQUAL]: |
| 68 if (inTag) { | 68 if (inTag) { |
| 69 if (_maybeEatChar(tmplTokens.tokens[TokenKind.SINGLE_QUOTE])) { | 69 if (_maybeEatChar(tmplTokens.tokens[TokenKind.SINGLE_QUOTE])) { |
| 70 return finishQuotedAttrValue( | 70 return finishQuotedAttrValue( |
| 71 tmplTokens.tokens[TokenKind.SINGLE_QUOTE]); | 71 tmplTokens.tokens[TokenKind.SINGLE_QUOTE]); |
| 72 } else if (_maybeEatChar(tmplTokens.tokens[TokenKind.DOUBLE_QUOTE])) { | 72 } else if (_maybeEatChar(tmplTokens.tokens[TokenKind.DOUBLE_QUOTE])) { |
| 73 return finishQuotedAttrValue( | 73 return finishQuotedAttrValue( |
| 74 tmplTokens.tokens[TokenKind.DOUBLE_QUOTE]); | 74 tmplTokens.tokens[TokenKind.DOUBLE_QUOTE]); |
| 75 } else if (isAttributeValueStart(_peekChar())) { | 75 } else if (TokenizerHelpers.isAttributeValueStart(_peekChar())) { |
| 76 return finishAttrValue(); | 76 return finishAttrValue(); |
| 77 } | 77 } |
| 78 } | 78 } |
| 79 return _finishToken(TokenKind.EQUAL); | 79 return _finishToken(TokenKind.EQUAL); |
| 80 case tmplTokens.tokens[TokenKind.SLASH]: | 80 case tmplTokens.tokens[TokenKind.SLASH]: |
| 81 if (_maybeEatChar(tmplTokens.tokens[TokenKind.GREATER_THAN])) { | 81 if (_maybeEatChar(tmplTokens.tokens[TokenKind.GREATER_THAN])) { |
| 82 return _finishToken(TokenKind.END_NO_SCOPE_TAG); // /> | 82 return _finishToken(TokenKind.END_NO_SCOPE_TAG); // /> |
| 83 } else if (_maybeEatChar(tmplTokens.tokens[TokenKind.ASTERISK])) { | 83 } else if (_maybeEatChar(tmplTokens.tokens[TokenKind.ASTERISK])) { |
| 84 return finishMultiLineComment(); | 84 return finishMultiLineComment(); |
| 85 } else { | 85 } else { |
| 86 return _finishToken(TokenKind.SLASH); | 86 return _finishToken(TokenKind.SLASH); |
| 87 } | 87 } |
| 88 case tmplTokens.tokens[TokenKind.DOLLAR]: | 88 case tmplTokens.tokens[TokenKind.DOLLAR]: |
| 89 if (_maybeEatChar(tmplTokens.tokens[TokenKind.LBRACE])) { | 89 if (_maybeEatChar(tmplTokens.tokens[TokenKind.LBRACE])) { |
| 90 if (_maybeEatChar(tmplTokens.tokens[TokenKind.HASH])) { | 90 if (_maybeEatChar(tmplTokens.tokens[TokenKind.HASH])) { |
| 91 return _finishToken(TokenKind.START_COMMAND); // ${# | 91 return _finishToken(TokenKind.START_COMMAND); // ${# |
| 92 } else if (_maybeEatChar(tmplTokens.tokens[TokenKind.SLASH])) { | 92 } else if (_maybeEatChar(tmplTokens.tokens[TokenKind.SLASH])) { |
| 93 return _finishToken(TokenKind.END_COMMAND); // ${/ | 93 return _finishToken(TokenKind.END_COMMAND); // ${/ |
| 94 } else { | 94 } else { |
| 95 return _finishToken(TokenKind.START_EXPRESSION); // ${ | 95 return _finishToken(TokenKind.START_EXPRESSION); // ${ |
| 96 } | 96 } |
| 97 } else { | 97 } else { |
| 98 return _finishToken(TokenKind.DOLLAR); | 98 return _finishToken(TokenKind.DOLLAR); |
| 99 } | 99 } |
| 100 | 100 |
| 101 default: | 101 default: |
| 102 if (TokenizerHelpers.isIdentifierStart(ch)) { | 102 if (TokenizerHelpers.isIdentifierStart(ch)) { |
| 103 return this.finishIdentifier(); | 103 return this.finishIdentifier(); |
| 104 } else if (isDigit(ch)) { | 104 } else if (TokenizerHelpers.isDigit(ch)) { |
| 105 return this.finishNumber(); | 105 return this.finishNumber(); |
| 106 } else { | 106 } else { |
| 107 return _errorToken(); | 107 return _errorToken(); |
| 108 } | 108 } |
| 109 } | 109 } |
| 110 } | 110 } |
| 111 | 111 |
| 112 // TODO(jmesserly): we need a way to emit human readable error messages from | 112 // TODO(jmesserly): we need a way to emit human readable error messages from |
| 113 // the tokenizer. | 113 // the tokenizer. |
| 114 Token _errorToken([String message = null]) { | 114 Token _errorToken([String message = null]) { |
| (...skipping 57 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 172 } else { | 172 } else { |
| 173 buf.add(ch); | 173 buf.add(ch); |
| 174 } | 174 } |
| 175 } | 175 } |
| 176 } | 176 } |
| 177 | 177 |
| 178 Token finishAttrValue() { | 178 Token finishAttrValue() { |
| 179 var buf = new List<int>(); | 179 var buf = new List<int>(); |
| 180 while (true) { | 180 while (true) { |
| 181 int ch = _peekChar(); | 181 int ch = _peekChar(); |
| 182 if (isWhitespace(ch) || isSlash(ch) || isCloseTag(ch)) { | 182 if (TokenizerHelpers.isWhitespace(ch) || TokenizerHelpers.isSlash(ch) || |
| 183 TokenizerHelpers.isCloseTag(ch)) { |
| 183 return _makeAttributeValueToken(buf); | 184 return _makeAttributeValueToken(buf); |
| 184 } else if (ch == 0) { | 185 } else if (ch == 0) { |
| 185 return _errorToken(); | 186 return _errorToken(); |
| 186 } else { | 187 } else { |
| 187 buf.add(_nextChar()); | 188 buf.add(_nextChar()); |
| 188 } | 189 } |
| 189 } | 190 } |
| 190 } | 191 } |
| 191 | 192 |
| 192 Token finishNumber() { | 193 Token finishNumber() { |
| 193 eatDigits(); | 194 eatDigits(); |
| 194 | 195 |
| 195 if (_peekChar() == 46/*.*/) { | 196 if (_peekChar() == 46/*.*/) { |
| 196 // Handle the case of 1.toString(). | 197 // Handle the case of 1.toString(). |
| 197 _nextChar(); | 198 _nextChar(); |
| 198 if (isDigit(_peekChar())) { | 199 if (TokenizerHelpers.isDigit(_peekChar())) { |
| 199 eatDigits(); | 200 eatDigits(); |
| 200 return _finishToken(TokenKind.DOUBLE); | 201 return _finishToken(TokenKind.DOUBLE); |
| 201 } else { | 202 } else { |
| 202 _index -= 1; | 203 _index -= 1; |
| 203 } | 204 } |
| 204 } | 205 } |
| 205 | 206 |
| 206 return _finishToken(TokenKind.INTEGER); | 207 return _finishToken(TokenKind.INTEGER); |
| 207 } | 208 } |
| 208 | 209 |
| 209 bool maybeEatDigit() { | 210 bool maybeEatDigit() { |
| 210 if (_index < _text.length && isDigit(_text.charCodeAt(_index))) { | 211 if (_index < _text.length && TokenizerHelpers.isDigit( |
| 212 _text.charCodeAt(_index))) { |
| 211 _index += 1; | 213 _index += 1; |
| 212 return true; | 214 return true; |
| 213 } | 215 } |
| 214 return false; | 216 return false; |
| 215 } | 217 } |
| 216 | 218 |
| 217 void eatHexDigits() { | 219 void eatHexDigits() { |
| 218 while (_index < _text.length) { | 220 while (_index < _text.length) { |
| 219 if (isHexDigit(_text.charCodeAt(_index))) { | 221 if (TokenizerHelpers.isHexDigit(_text.charCodeAt(_index))) { |
| 220 _index += 1; | 222 _index += 1; |
| 221 } else { | 223 } else { |
| 222 return; | 224 return; |
| 223 } | 225 } |
| 224 } | 226 } |
| 225 } | 227 } |
| 226 | 228 |
| 227 bool maybeEatHexDigit() { | 229 bool maybeEatHexDigit() { |
| 228 if (_index < _text.length && isHexDigit(_text.charCodeAt(_index))) { | 230 if (_index < _text.length && TokenizerHelpers.isHexDigit( |
| 231 _text.charCodeAt(_index))) { |
| 229 _index += 1; | 232 _index += 1; |
| 230 return true; | 233 return true; |
| 231 } | 234 } |
| 232 return false; | 235 return false; |
| 233 } | 236 } |
| 234 | 237 |
| 235 Token finishMultiLineComment() { | 238 Token finishMultiLineComment() { |
| 236 while (true) { | 239 while (true) { |
| 237 int ch = _nextChar(); | 240 int ch = _nextChar(); |
| 238 if (ch == 0) { | 241 if (ch == 0) { |
| (...skipping 60 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 299 } | 302 } |
| 300 | 303 |
| 301 static bool isSlash(int c) { | 304 static bool isSlash(int c) { |
| 302 return (c == 47/* / */); | 305 return (c == 47/* / */); |
| 303 } | 306 } |
| 304 | 307 |
| 305 static bool isCloseTag(int c) { | 308 static bool isCloseTag(int c) { |
| 306 return (c == 62/* > */); | 309 return (c == 62/* > */); |
| 307 } | 310 } |
| 308 } | 311 } |
| OLD | NEW |