| OLD | NEW |
| 1 // Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 // TODO(terry): Need to be consistent with tokens either they're ASCII tokens | 5 // TODO(terry): Need to be consistent with tokens either they're ASCII tokens |
| 6 // e.g., ASTERISK or they're CSS e.g., PSEUDO, COMBINATOR_*. | 6 // e.g., ASTERISK or they're CSS e.g., PSEUDO, COMBINATOR_*. |
| 7 class TokenKind { | 7 class TokenKind { |
| 8 // Common shared tokens used in TokenizerBase. | 8 // Common shared tokens used in TokenizerBase. |
| 9 static final int UNUSED = 0; // Unused place holder... | 9 static final int UNUSED = 0; // Unused place holder... |
| 10 static final int END_OF_FILE = 1; | 10 static final int END_OF_FILE = 1; |
| (...skipping 14 matching lines...) Expand all Loading... |
| 25 // Unique tokens. | 25 // Unique tokens. |
| 26 static final int LESS_THAN = 15; // < | 26 static final int LESS_THAN = 15; // < |
| 27 static final int GREATER_THAN = 16; // > | 27 static final int GREATER_THAN = 16; // > |
| 28 static final int SLASH = 17; // / | 28 static final int SLASH = 17; // / |
| 29 static final int DOLLAR = 18; // $ | 29 static final int DOLLAR = 18; // $ |
| 30 static final int HASH = 19; // # | 30 static final int HASH = 19; // # |
| 31 static final int MINUS = 20; // - | 31 static final int MINUS = 20; // - |
| 32 static final int EQUAL = 21; // = | 32 static final int EQUAL = 21; // = |
| 33 static final int DOUBLE_QUOTE = 22; // " | 33 static final int DOUBLE_QUOTE = 22; // " |
| 34 static final int SINGLE_QUOTE = 23; // ' | 34 static final int SINGLE_QUOTE = 23; // ' |
| 35 static final int ASTERISK = 24; // * |
| 35 | 36 |
| 36 // WARNING: END_TOKENS must be 1 greater than the last token above (last | 37 // WARNING: END_TOKENS must be 1 greater than the last token above (last |
| 37 // character in our list). Also add to kindToString function and the | 38 // character in our list). Also add to kindToString function and the |
| 38 // constructor for TokenKind. | 39 // constructor for TokenKind. |
| 39 | 40 |
| 40 static final int END_TOKENS = 24; // Marker for last token in list | 41 static final int END_TOKENS = 25; // Marker for last token in list |
| 41 | 42 |
| 42 // Synthesized tokens: | 43 // Synthesized tokens: |
| 43 | 44 |
| 44 static final int END_NO_SCOPE_TAG = 50; // /> | 45 static final int END_NO_SCOPE_TAG = 50; // /> |
| 45 static final int START_EXPRESSION = 51; // ${ | 46 static final int START_EXPRESSION = 51; // ${ |
| 46 static final int START_COMMAND = 52; // ${# | 47 static final int START_COMMAND = 52; // ${# |
| 47 static final int END_COMMAND = 53; // ${/ | 48 static final int END_COMMAND = 53; // ${/ |
| 48 static final int EACH_COMMAND = 53; // ${#each list} | 49 static final int EACH_COMMAND = 53; // ${#each list} |
| 49 static final int WITH_COMMAND = 54; // ${#with object} | 50 static final int WITH_COMMAND = 54; // ${#with object} |
| 50 static final int IF_COMMAND = 55; // ${#if (expression)} | 51 static final int IF_COMMAND = 55; // ${#if (expression)} |
| (...skipping 136 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 187 | 188 |
| 188 static bool validTagName(int tokId) { | 189 static bool validTagName(int tokId) { |
| 189 return tokId >= TokenKind.START_HTML_ELEMENT && | 190 return tokId >= TokenKind.START_HTML_ELEMENT && |
| 190 tokId <= TokenKind.END_HTML_ELEMENT; | 191 tokId <= TokenKind.END_HTML_ELEMENT; |
| 191 } | 192 } |
| 192 | 193 |
| 193 static final List<Map<int, String>> _KEYWORDS = const [ | 194 static final List<Map<int, String>> _KEYWORDS = const [ |
| 194 const {'type': TokenKind.TEMPLATE_KEYWORD, 'value' : 'template'}, | 195 const {'type': TokenKind.TEMPLATE_KEYWORD, 'value' : 'template'}, |
| 195 ]; | 196 ]; |
| 196 | 197 |
| 198 static final List<int> _NON_SCOPED_ELEMENTS = const [ |
| 199 BR_ELEMENT, |
| 200 ]; |
| 201 |
| 202 // tag values starting with a minus sign implies tag can be unscoped e.g., |
| 203 // <br> is valid without <br></br> or <br/> |
| 197 static final List<Map<int, String>> _ELEMENTS = const [ | 204 static final List<Map<int, String>> _ELEMENTS = const [ |
| 198 const {'type': TokenKind.A_ELEMENT, 'value' : 'a'}, | 205 const {'type': TokenKind.A_ELEMENT, 'value' : 'a'}, |
| 199 const {'type': TokenKind.ABBR_ELEMENT, 'value' : 'abbr'}, | 206 const {'type': TokenKind.ABBR_ELEMENT, 'value' : 'abbr'}, |
| 200 const {'type': TokenKind.ACRONYM_ELEMENT, 'value' : 'acronym'}, | 207 const {'type': TokenKind.ACRONYM_ELEMENT, 'value' : 'acronym'}, |
| 201 const {'type': TokenKind.ADDRESS_ELEMENT, 'value' : 'address'}, | 208 const {'type': TokenKind.ADDRESS_ELEMENT, 'value' : 'address'}, |
| 202 const {'type': TokenKind.APPLET_ELEMENT, 'value' : 'applet'}, | 209 const {'type': TokenKind.APPLET_ELEMENT, 'value' : 'applet'}, |
| 203 const {'type': TokenKind.AREA_ELEMENT, 'value' : 'area'}, | 210 const {'type': TokenKind.AREA_ELEMENT, 'value' : 'area'}, |
| 204 const {'type': TokenKind.B_ELEMENT, 'value' : 'b'}, | 211 const {'type': TokenKind.B_ELEMENT, 'value' : 'b'}, |
| 205 const {'type': TokenKind.BASE_ELEMENT, 'value' : 'base'}, | 212 const {'type': TokenKind.BASE_ELEMENT, 'value' : 'base'}, |
| 206 const {'type': TokenKind.BASEFONT_ELEMENT, 'value' : 'basefont'}, | 213 const {'type': TokenKind.BASEFONT_ELEMENT, 'value' : 'basefont'}, |
| (...skipping 112 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 319 if (match) { | 326 if (match) { |
| 320 // Completely matched; return the token for this unit. | 327 // Completely matched; return the token for this unit. |
| 321 return entry[tokenField]; | 328 return entry[tokenField]; |
| 322 } | 329 } |
| 323 } | 330 } |
| 324 } | 331 } |
| 325 | 332 |
| 326 return -1; // Not a unit token. | 333 return -1; // Not a unit token. |
| 327 } | 334 } |
| 328 | 335 |
| 329 /* Map _ELEMENTS type to the real name. */ | |
| 330 static String elementsToName(int kind) { | |
| 331 for (final entry in TokenKind._ELEMENTS) { | |
| 332 if (kind == entry['type']) { | |
| 333 return entry['value']; | |
| 334 } | |
| 335 } | |
| 336 } | |
| 337 | |
| 338 /* | 336 /* |
| 339 * Return the token that matches the element ident found. | 337 * Return the token that matches the element ident found. |
| 340 */ | 338 */ |
| 341 static int matchElements(String text, int offset, int length) { | 339 static int matchElements(String text, int offset, int length) { |
| 342 return matchList(_ELEMENTS, 'type', text, offset, length); | 340 return matchList(_ELEMENTS, 'type', text, offset, length); |
| 343 } | 341 } |
| 344 | 342 |
| 345 static String tagNameFromTokenId(int tagTokenId) { | 343 static String tagNameFromTokenId(int tagTokenId) { |
| 346 if (tagTokenId >= TokenKind.START_HTML_ELEMENT && | 344 if (TokenKind.validTagName(tagTokenId)) { |
| 347 tagTokenId <= TokenKind.END_HTML_ELEMENT) { | |
| 348 for (final tag in TokenKind._ELEMENTS) { | 345 for (final tag in TokenKind._ELEMENTS) { |
| 349 if (tag['type'] == tagTokenId) { | 346 if (tag['type'] == tagTokenId) { |
| 350 return tag['value']; | 347 return tag['value']; |
| 351 } | 348 } |
| 352 } | 349 } |
| 353 } | 350 } |
| 354 } | 351 } |
| 355 | 352 |
| 353 static bool unscopedTag(int tagTokenId) { |
| 354 for (final tagId in TokenKind._NON_SCOPED_ELEMENTS) { |
| 355 if (tagId == tagTokenId) { |
| 356 return true; |
| 357 } |
| 358 } |
| 359 |
| 360 return false; |
| 361 } |
| 362 |
| 356 static int matchKeywords(String text, int offset, int length) { | 363 static int matchKeywords(String text, int offset, int length) { |
| 357 return matchList(_KEYWORDS, 'type', text, offset, length); | 364 return matchList(_KEYWORDS, 'type', text, offset, length); |
| 358 } | 365 } |
| 359 | 366 |
| 360 static String kindToString(int kind) { | 367 static String kindToString(int kind) { |
| 361 switch(kind) { | 368 switch(kind) { |
| 362 case TokenKind.UNUSED: return "ERROR"; | 369 case TokenKind.UNUSED: return "ERROR"; |
| 363 case TokenKind.END_OF_FILE: return "end of file"; | 370 case TokenKind.END_OF_FILE: return "end of file"; |
| 364 case TokenKind.LPAREN: return "("; | 371 case TokenKind.LPAREN: return "("; |
| 365 case TokenKind.RPAREN: return ")"; | 372 case TokenKind.RPAREN: return ")"; |
| (...skipping 10 matching lines...) Expand all Loading... |
| 376 case TokenKind.COMMA: return ","; | 383 case TokenKind.COMMA: return ","; |
| 377 case TokenKind.LESS_THAN: return "<"; | 384 case TokenKind.LESS_THAN: return "<"; |
| 378 case TokenKind.GREATER_THAN: return ">"; | 385 case TokenKind.GREATER_THAN: return ">"; |
| 379 case TokenKind.SLASH: return "/"; | 386 case TokenKind.SLASH: return "/"; |
| 380 case TokenKind.DOLLAR: return "\$"; | 387 case TokenKind.DOLLAR: return "\$"; |
| 381 case TokenKind.HASH: return "#"; | 388 case TokenKind.HASH: return "#"; |
| 382 case TokenKind.MINUS: return '-'; | 389 case TokenKind.MINUS: return '-'; |
| 383 case TokenKind.EQUAL: return '='; | 390 case TokenKind.EQUAL: return '='; |
| 384 case TokenKind.DOUBLE_QUOTE: return '"'; | 391 case TokenKind.DOUBLE_QUOTE: return '"'; |
| 385 case TokenKind.SINGLE_QUOTE: return "'"; | 392 case TokenKind.SINGLE_QUOTE: return "'"; |
| 393 case TokenKind.ASTERISK: return "*"; |
| 386 case TokenKind.END_NO_SCOPE_TAG: return '/>'; | 394 case TokenKind.END_NO_SCOPE_TAG: return '/>'; |
| 387 case TokenKind.START_EXPRESSION: return '\${'; | 395 case TokenKind.START_EXPRESSION: return '\${'; |
| 388 case TokenKind.START_COMMAND: return '\${#'; | 396 case TokenKind.START_COMMAND: return '\${#'; |
| 389 case TokenKind.END_COMMAND: return '\${/'; | 397 case TokenKind.END_COMMAND: return '\${/'; |
| 390 case TokenKind.EACH_COMMAND: return '\${#each list}'; | 398 case TokenKind.EACH_COMMAND: return '\${#each list}'; |
| 391 case TokenKind.WITH_COMMAND: return '\${with object}'; | 399 case TokenKind.WITH_COMMAND: return '\${with object}'; |
| 392 case TokenKind.IF_COMMAND: return '\${#if (expression)}'; | 400 case TokenKind.IF_COMMAND: return '\${#if (expression)}'; |
| 393 case TokenKind.ELSE_COMMAND: return '\${#end}'; | 401 case TokenKind.ELSE_COMMAND: return '\${#end}'; |
| 394 case TokenKind.INTEGER: return 'integer'; | 402 case TokenKind.INTEGER: return 'integer'; |
| 395 case TokenKind.DOUBLE: return 'double'; | 403 case TokenKind.DOUBLE: return 'double'; |
| (...skipping 36 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 432 tokens.add(TokenKind.kindToString(TokenKind.COMMA).charCodeAt(0)); | 440 tokens.add(TokenKind.kindToString(TokenKind.COMMA).charCodeAt(0)); |
| 433 tokens.add(TokenKind.kindToString(TokenKind.LESS_THAN).charCodeAt(0)); | 441 tokens.add(TokenKind.kindToString(TokenKind.LESS_THAN).charCodeAt(0)); |
| 434 tokens.add(TokenKind.kindToString(TokenKind.GREATER_THAN).charCodeAt(0)); | 442 tokens.add(TokenKind.kindToString(TokenKind.GREATER_THAN).charCodeAt(0)); |
| 435 tokens.add(TokenKind.kindToString(TokenKind.SLASH).charCodeAt(0)); | 443 tokens.add(TokenKind.kindToString(TokenKind.SLASH).charCodeAt(0)); |
| 436 tokens.add(TokenKind.kindToString(TokenKind.DOLLAR).charCodeAt(0)); | 444 tokens.add(TokenKind.kindToString(TokenKind.DOLLAR).charCodeAt(0)); |
| 437 tokens.add(TokenKind.kindToString(TokenKind.HASH).charCodeAt(0)); | 445 tokens.add(TokenKind.kindToString(TokenKind.HASH).charCodeAt(0)); |
| 438 tokens.add(TokenKind.kindToString(TokenKind.MINUS).charCodeAt(0)); | 446 tokens.add(TokenKind.kindToString(TokenKind.MINUS).charCodeAt(0)); |
| 439 tokens.add(TokenKind.kindToString(TokenKind.EQUAL).charCodeAt(0)); | 447 tokens.add(TokenKind.kindToString(TokenKind.EQUAL).charCodeAt(0)); |
| 440 tokens.add(TokenKind.kindToString(TokenKind.DOUBLE_QUOTE).charCodeAt(0)); | 448 tokens.add(TokenKind.kindToString(TokenKind.DOUBLE_QUOTE).charCodeAt(0)); |
| 441 tokens.add(TokenKind.kindToString(TokenKind.SINGLE_QUOTE).charCodeAt(0)); | 449 tokens.add(TokenKind.kindToString(TokenKind.SINGLE_QUOTE).charCodeAt(0)); |
| 450 tokens.add(TokenKind.kindToString(TokenKind.ASTERISK).charCodeAt(0)); |
| 442 | 451 |
| 443 assert(tokens.length == TokenKind.END_TOKENS); | 452 assert(tokens.length == TokenKind.END_TOKENS); |
| 444 } | 453 } |
| 445 | 454 |
| 446 static bool isIdentifier(int kind) { | 455 static bool isIdentifier(int kind) { |
| 447 return kind == IDENTIFIER; | 456 return kind == IDENTIFIER; |
| 448 } | 457 } |
| 449 } | 458 } |
| 450 | 459 |
| 451 class NoElementMatchException implements Exception { | 460 class NoElementMatchException implements Exception { |
| 452 String _tagName; | 461 String _tagName; |
| 453 NoElementMatchException(this._tagName); | 462 NoElementMatchException(this._tagName); |
| 454 | 463 |
| 455 String get name() => _tagName; | 464 String get name() => _tagName; |
| 456 } | 465 } |
| OLD | NEW |