| OLD | NEW |
| 1 // Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 class Tokenizer extends CSSTokenizerBase { | 5 class Tokenizer extends CSSTokenizerBase { |
| 6 TokenKind cssTokens; | 6 TokenKind cssTokens; |
| 7 | 7 |
| 8 bool _selectorParsing; | 8 bool _selectorParsing; |
| 9 | 9 |
| 10 Tokenizer(SourceFile source, bool skipWhitespace, [int index = 0]) | 10 Tokenizer(SourceFile source, bool skipWhitespace, [int index = 0]) |
| (...skipping 140 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 151 return _finishToken(TokenKind.SUFFIX_MATCH); // $= | 151 return _finishToken(TokenKind.SUFFIX_MATCH); // $= |
| 152 } else { | 152 } else { |
| 153 return _finishToken(TokenKind.DOLLAR); | 153 return _finishToken(TokenKind.DOLLAR); |
| 154 } | 154 } |
| 155 case cssTokens.tokens[TokenKind.BANG]: | 155 case cssTokens.tokens[TokenKind.BANG]: |
| 156 Token tok = finishIdentifier(ch); | 156 Token tok = finishIdentifier(ch); |
| 157 return (tok == null) ? _finishToken(TokenKind.BANG) : tok; | 157 return (tok == null) ? _finishToken(TokenKind.BANG) : tok; |
| 158 default: | 158 default: |
| 159 if (TokenizerHelpers.isIdentifierStart(ch)) { | 159 if (TokenizerHelpers.isIdentifierStart(ch)) { |
| 160 return this.finishIdentifier(ch); | 160 return this.finishIdentifier(ch); |
| 161 } else if (isDigit(ch)) { | 161 } else if (TokenizerHelpers.isDigit(ch)) { |
| 162 return this.finishNumber(); | 162 return this.finishNumber(); |
| 163 } else { | 163 } else { |
| 164 return _errorToken(); | 164 return _errorToken(); |
| 165 } | 165 } |
| 166 } | 166 } |
| 167 } | 167 } |
| 168 | 168 |
| 169 // TODO(jmesserly): we need a way to emit human readable error messages from | 169 // TODO(jmesserly): we need a way to emit human readable error messages from |
| 170 // the tokenizer. | 170 // the tokenizer. |
| 171 Token _errorToken([String message = null]) { | 171 Token _errorToken([String message = null]) { |
| 172 return _finishToken(TokenKind.ERROR); | 172 return _finishToken(TokenKind.ERROR); |
| 173 } | 173 } |
| 174 | 174 |
| 175 int getIdentifierKind() { | 175 int getIdentifierKind() { |
| 176 // Is the identifier a unit type? | 176 // Is the identifier a unit type? |
| 177 int tokId = TokenKind.matchUnits(_text, _startIndex, _index - _startIndex); | 177 int tokId = TokenKind.matchUnits(_text, _startIndex, _index - _startIndex); |
| 178 if (tokId == -1) { | 178 if (tokId == -1) { |
| 179 // No, is it a directive? | 179 // No, is it a directive? |
| 180 tokId = TokenKind.matchDirectives(_text, _startIndex, _index - _startIndex
); | 180 tokId = TokenKind.matchDirectives( |
| 181 _text, _startIndex, _index - _startIndex); |
| 181 } | 182 } |
| 182 if (tokId == -1) { | 183 if (tokId == -1) { |
| 183 tokId = (_text.substring(_startIndex, _index) == '!important') ? | 184 tokId = (_text.substring(_startIndex, _index) == '!important') ? |
| 184 TokenKind.IMPORTANT : -1; | 185 TokenKind.IMPORTANT : -1; |
| 185 } | 186 } |
| 186 | 187 |
| 187 return tokId >= 0 ? tokId : TokenKind.IDENTIFIER; | 188 return tokId >= 0 ? tokId : TokenKind.IDENTIFIER; |
| 188 } | 189 } |
| 189 | 190 |
| 190 // Need to override so CSS version of isIdentifierPart is used. | 191 // Need to override so CSS version of isIdentifierPart is used. |
| (...skipping 21 matching lines...) Expand all Loading... |
| 212 Token finishImportant() { | 213 Token finishImportant() { |
| 213 | 214 |
| 214 } | 215 } |
| 215 | 216 |
| 216 Token finishNumber() { | 217 Token finishNumber() { |
| 217 eatDigits(); | 218 eatDigits(); |
| 218 | 219 |
| 219 if (_peekChar() == 46/*.*/) { | 220 if (_peekChar() == 46/*.*/) { |
| 220 // Handle the case of 1.toString(). | 221 // Handle the case of 1.toString(). |
| 221 _nextChar(); | 222 _nextChar(); |
| 222 if (isDigit(_peekChar())) { | 223 if (TokenizerHelpers.isDigit(_peekChar())) { |
| 223 eatDigits(); | 224 eatDigits(); |
| 224 return _finishToken(TokenKind.DOUBLE); | 225 return _finishToken(TokenKind.DOUBLE); |
| 225 } else { | 226 } else { |
| 226 _index -= 1; | 227 _index -= 1; |
| 227 } | 228 } |
| 228 } | 229 } |
| 229 | 230 |
| 230 return _finishToken(TokenKind.INTEGER); | 231 return _finishToken(TokenKind.INTEGER); |
| 231 } | 232 } |
| 232 | 233 |
| 233 bool maybeEatDigit() { | 234 bool maybeEatDigit() { |
| 234 if (_index < _text.length && isDigit(_text.charCodeAt(_index))) { | 235 if (_index < _text.length |
| 236 && TokenizerHelpers.isDigit(_text.charCodeAt(_index))) { |
| 235 _index += 1; | 237 _index += 1; |
| 236 return true; | 238 return true; |
| 237 } | 239 } |
| 238 return false; | 240 return false; |
| 239 } | 241 } |
| 240 | 242 |
| 241 void eatHexDigits() { | 243 void eatHexDigits() { |
| 242 while (_index < _text.length) { | 244 while (_index < _text.length) { |
| 243 if (isHexDigit(_text.charCodeAt(_index))) { | 245 if (TokenizerHelpers.isHexDigit(_text.charCodeAt(_index))) { |
| 244 _index += 1; | 246 _index += 1; |
| 245 } else { | 247 } else { |
| 246 return; | 248 return; |
| 247 } | 249 } |
| 248 } | 250 } |
| 249 } | 251 } |
| 250 | 252 |
| 251 bool maybeEatHexDigit() { | 253 bool maybeEatHexDigit() { |
| 252 if (_index < _text.length && isHexDigit(_text.charCodeAt(_index))) { | 254 if (_index < _text.length |
| 255 && TokenizerHelpers.isHexDigit(_text.charCodeAt(_index))) { |
| 253 _index += 1; | 256 _index += 1; |
| 254 return true; | 257 return true; |
| 255 } | 258 } |
| 256 return false; | 259 return false; |
| 257 } | 260 } |
| 258 | 261 |
| 259 Token finishMultiLineComment() { | 262 Token finishMultiLineComment() { |
| 260 while (true) { | 263 while (true) { |
| 261 int ch = _nextChar(); | 264 int ch = _nextChar(); |
| 262 if (ch == 0) { | 265 if (ch == 0) { |
| (...skipping 31 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 294 static bool isIdentifierStart(int c) { | 297 static bool isIdentifierStart(int c) { |
| 295 return ((c >= 97/*a*/ && c <= 122/*z*/) || (c >= 65/*A*/ && c <= 90/*Z*/) || | 298 return ((c >= 97/*a*/ && c <= 122/*z*/) || (c >= 65/*A*/ && c <= 90/*Z*/) || |
| 296 c == 95/*_*/ || c == 45 /*-*/); | 299 c == 95/*_*/ || c == 45 /*-*/); |
| 297 } | 300 } |
| 298 | 301 |
| 299 static bool isDigit(int c) { | 302 static bool isDigit(int c) { |
| 300 return (c >= 48/*0*/ && c <= 57/*9*/); | 303 return (c >= 48/*0*/ && c <= 57/*9*/); |
| 301 } | 304 } |
| 302 | 305 |
| 303 static bool isHexDigit(int c) { | 306 static bool isHexDigit(int c) { |
| 304 return (isDigit(c) || (c >= 97/*a*/ && c <= 102/*f*/) || (c >= 65/*A*/ && c
<= 70/*F*/)); | 307 return (isDigit(c) || (c >= 97/*a*/ && c <= 102/*f*/) |
| 305 } | 308 || (c >= 65/*A*/ && c <= 70/*F*/)); |
| 306 | |
| 307 static bool isWhitespace(int c) { | |
| 308 return (c == 32/*' '*/ || c == 9/*'\t'*/ || c == 10/*'\n'*/ || c == 13/*'\r'
*/); | |
| 309 } | 309 } |
| 310 | 310 |
| 311 static bool isIdentifierPart(int c) { | 311 static bool isIdentifierPart(int c) { |
| 312 return (isIdentifierStart(c) || isDigit(c) || c == 45 /*-*/); | 312 return (isIdentifierStart(c) || isDigit(c) || c == 45 /*-*/); |
| 313 } | 313 } |
| 314 | |
| 315 static bool isInterpIdentifierPart(int c) { | |
| 316 return (isIdentifierStart(c) || isDigit(c)); | |
| 317 } | |
| 318 } | 314 } |
| 319 | 315 |
| OLD | NEW |