| OLD | NEW |
| 1 // Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 // Generated by scripts/tokenizer_gen.py. | 4 // Generated by scripts/tokenizer_gen.py. |
| 5 | 5 |
| 6 | 6 |
| 7 interface TokenSource { | 7 interface TokenSource { |
| 8 Token next(); | 8 Token next(); |
| 9 } | 9 } |
| 10 | 10 |
| (...skipping 13 matching lines...) Expand all Loading... |
| 24 var newStack = new InterpStack(stack, quote, isMultiline); | 24 var newStack = new InterpStack(stack, quote, isMultiline); |
| 25 if (stack != null) newStack.previous = stack; | 25 if (stack != null) newStack.previous = stack; |
| 26 return newStack; | 26 return newStack; |
| 27 } | 27 } |
| 28 } | 28 } |
| 29 | 29 |
| 30 /** | 30 /** |
| 31 * The base class for our tokenizer. The hand coded parts are in this file, with | 31 * The base class for our tokenizer. The hand coded parts are in this file, with |
| 32 * the generated parts in the subclass Tokenizer. | 32 * the generated parts in the subclass Tokenizer. |
| 33 */ | 33 */ |
| 34 class CSSTokenizerBase extends TokenizerHelpers implements TokenSource { | 34 class CSSTokenizerBase implements TokenSource { |
| 35 final SourceFile _source; | 35 final SourceFile _source; |
| 36 final bool _skipWhitespace; | 36 final bool _skipWhitespace; |
| 37 String _text; | 37 String _text; |
| 38 | 38 |
| 39 int _index; | 39 int _index; |
| 40 int _startIndex; | 40 int _startIndex; |
| 41 | 41 |
| 42 /** Keeps track of string interpolation state. */ | 42 /** Keeps track of string interpolation state. */ |
| 43 InterpStack _interpStack; | 43 InterpStack _interpStack; |
| 44 | 44 |
| (...skipping 106 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 151 | 151 |
| 152 if (_skipWhitespace) { | 152 if (_skipWhitespace) { |
| 153 return next(); | 153 return next(); |
| 154 } else { | 154 } else { |
| 155 return _finishToken(TokenKind.COMMENT); | 155 return _finishToken(TokenKind.COMMENT); |
| 156 } | 156 } |
| 157 } | 157 } |
| 158 | 158 |
| 159 void eatDigits() { | 159 void eatDigits() { |
| 160 while (_index < _text.length) { | 160 while (_index < _text.length) { |
| 161 if (isDigit(_text.charCodeAt(_index))) { | 161 if (TokenizerHelpers.isDigit(_text.charCodeAt(_index))) { |
| 162 _index++; | 162 _index++; |
| 163 } else { | 163 } else { |
| 164 return; | 164 return; |
| 165 } | 165 } |
| 166 } | 166 } |
| 167 } | 167 } |
| 168 | 168 |
| 169 static int _hexDigit(int c) { | 169 static int _hexDigit(int c) { |
| 170 if(c >= 48/*0*/ && c <= 57/*9*/) { | 170 if(c >= 48/*0*/ && c <= 57/*9*/) { |
| 171 return c - 48; | 171 return c - 48; |
| (...skipping 36 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 208 | 208 |
| 209 return result; | 209 return result; |
| 210 } | 210 } |
| 211 | 211 |
| 212 Token finishNumber() { | 212 Token finishNumber() { |
| 213 eatDigits(); | 213 eatDigits(); |
| 214 | 214 |
| 215 if (_peekChar() == 46/*.*/) { | 215 if (_peekChar() == 46/*.*/) { |
| 216 // Handle the case of 1.toString(). | 216 // Handle the case of 1.toString(). |
| 217 _nextChar(); | 217 _nextChar(); |
| 218 if (isDigit(_peekChar())) { | 218 if (TokenizerHelpers.isDigit(_peekChar())) { |
| 219 eatDigits(); | 219 eatDigits(); |
| 220 return finishNumberExtra(TokenKind.DOUBLE); | 220 return finishNumberExtra(TokenKind.DOUBLE); |
| 221 } else { | 221 } else { |
| 222 _index--; | 222 _index--; |
| 223 } | 223 } |
| 224 } | 224 } |
| 225 | 225 |
| 226 return finishNumberExtra(TokenKind.INTEGER); | 226 return finishNumberExtra(TokenKind.INTEGER); |
| 227 } | 227 } |
| 228 | 228 |
| 229 Token finishNumberExtra(int kind) { | 229 Token finishNumberExtra(int kind) { |
| 230 if (_maybeEatChar(101/*e*/) || _maybeEatChar(69/*E*/)) { | 230 if (_maybeEatChar(101/*e*/) || _maybeEatChar(69/*E*/)) { |
| 231 kind = TokenKind.DOUBLE; | 231 kind = TokenKind.DOUBLE; |
| 232 _maybeEatChar(45/*-*/); | 232 _maybeEatChar(45/*-*/); |
| 233 _maybeEatChar(43/*+*/); | 233 _maybeEatChar(43/*+*/); |
| 234 eatDigits(); | 234 eatDigits(); |
| 235 } | 235 } |
| 236 if (_peekChar() != 0 && isIdentifierStart(_peekChar())) { | 236 if (_peekChar() != 0 && TokenizerHelpers.isIdentifierStart(_peekChar())) { |
| 237 _nextChar(); | 237 _nextChar(); |
| 238 return _errorToken("illegal character in number"); | 238 return _errorToken("illegal character in number"); |
| 239 } | 239 } |
| 240 | 240 |
| 241 return _finishToken(kind); | 241 return _finishToken(kind); |
| 242 } | 242 } |
| 243 | 243 |
| 244 Token _makeStringToken(List<int> buf, bool isPart) { | 244 Token _makeStringToken(List<int> buf, bool isPart) { |
| 245 final s = new String.fromCharCodes(buf); | 245 final s = new String.fromCharCodes(buf); |
| 246 final kind = isPart ? TokenKind.STRING_PART : TokenKind.STRING; | 246 final kind = isPart ? TokenKind.STRING_PART : TokenKind.STRING; |
| (...skipping 174 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 421 return hexValue; | 421 return hexValue; |
| 422 } else if (hexValue <= 0x10FFFF){ | 422 } else if (hexValue <= 0x10FFFF){ |
| 423 world.fatal('unicode values greater than 2 bytes not implemented yet'); | 423 world.fatal('unicode values greater than 2 bytes not implemented yet'); |
| 424 return -1; | 424 return -1; |
| 425 } else { | 425 } else { |
| 426 return -1; | 426 return -1; |
| 427 } | 427 } |
| 428 } | 428 } |
| 429 | 429 |
| 430 Token finishDot() { | 430 Token finishDot() { |
| 431 if (isDigit(_peekChar())) { | 431 if (TokenizerHelpers.isDigit(_peekChar())) { |
| 432 eatDigits(); | 432 eatDigits(); |
| 433 return finishNumberExtra(TokenKind.DOUBLE); | 433 return finishNumberExtra(TokenKind.DOUBLE); |
| 434 } else { | 434 } else { |
| 435 return _finishToken(TokenKind.DOT); | 435 return _finishToken(TokenKind.DOT); |
| 436 } | 436 } |
| 437 } | 437 } |
| 438 | 438 |
| 439 Token finishIdentifier(int ch) { | 439 Token finishIdentifier(int ch) { |
| 440 while (_index < _text.length) { | 440 while (_index < _text.length) { |
| 441 if (!isIdentifierPart(_text.charCodeAt(_index++))) { | 441 if (!TokenizerHelpers.isIdentifierPart(_text.charCodeAt(_index++))) { |
| 442 _index--; | 442 _index--; |
| 443 break; | 443 break; |
| 444 } | 444 } |
| 445 } | 445 } |
| 446 int kind = getIdentifierKind(); | 446 int kind = getIdentifierKind(); |
| 447 if (kind == TokenKind.IDENTIFIER) { | 447 if (kind == TokenKind.IDENTIFIER) { |
| 448 return _finishToken(TokenKind.IDENTIFIER); | 448 return _finishToken(TokenKind.IDENTIFIER); |
| 449 } else { | 449 } else { |
| 450 return _finishToken(kind); | 450 return _finishToken(kind); |
| 451 } | 451 } |
| 452 } | 452 } |
| 453 } | 453 } |
| 454 | 454 |
| OLD | NEW |