| OLD | NEW |
| 1 // Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 interface Scanner { | 5 interface Scanner { |
| 6 Token tokenize(); | 6 Token tokenize(); |
| 7 } | 7 } |
| 8 | 8 |
| 9 /** | 9 /** |
| 10 * Common base class for a Dart scanner. | 10 * Common base class for a Dart scanner. |
| (...skipping 184 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 195 // TODO(ahe): Would a range check be faster? | 195 // TODO(ahe): Would a range check be faster? |
| 196 if (next === $1 || next === $2 || next === $3 || next === $4 || next === $5 | 196 if (next === $1 || next === $2 || next === $3 || next === $4 || next === $5 |
| 197 || next === $6 || next === $7 || next === $8 || next === $9) { | 197 || next === $6 || next === $7 || next === $8 || next === $9) { |
| 198 return tokenizeNumber(next); | 198 return tokenizeNumber(next); |
| 199 } | 199 } |
| 200 | 200 |
| 201 if (next === $EOF) { | 201 if (next === $EOF) { |
| 202 return $EOF; | 202 return $EOF; |
| 203 } | 203 } |
| 204 if (next < 0x1f) { | 204 if (next < 0x1f) { |
| 205 throw new MalformedInputException(charOffset); | 205 throw new MalformedInputException("illegal character $next", charOffset); |
| 206 } | 206 } |
| 207 // Non-ascii identifier. | 207 // Non-ascii identifier. |
| 208 return tokenizeIdentifier(next, byteOffset, true); | 208 return tokenizeIdentifier(next, byteOffset, true); |
| 209 } | 209 } |
| 210 | 210 |
| 211 int tokenizeTag(int next) { | 211 int tokenizeTag(int next) { |
| 212 // # or #!.*[\n\r] | 212 // # or #!.*[\n\r] |
| 213 if (byteOffset === 0) { | 213 if (byteOffset === 0) { |
| 214 if (peek() === $BANG) { | 214 if (peek() === $BANG) { |
| 215 do { | 215 do { |
| (...skipping 201 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 417 int start = byteOffset - 1; | 417 int start = byteOffset - 1; |
| 418 bool hasDigits = false; | 418 bool hasDigits = false; |
| 419 while (true) { | 419 while (true) { |
| 420 next = advance(); | 420 next = advance(); |
| 421 if (($0 <= next && next <= $9) | 421 if (($0 <= next && next <= $9) |
| 422 || ($A <= next && next <= $F) | 422 || ($A <= next && next <= $F) |
| 423 || ($a <= next && next <= $f)) { | 423 || ($a <= next && next <= $f)) { |
| 424 hasDigits = true; | 424 hasDigits = true; |
| 425 } else { | 425 } else { |
| 426 if (!hasDigits) { | 426 if (!hasDigits) { |
| 427 throw new MalformedInputException(charOffset); | 427 throw new MalformedInputException("hex digit expected", charOffset); |
| 428 } | 428 } |
| 429 appendByteStringToken(HEXADECIMAL_INFO, asciiString(start, 0)); | 429 appendByteStringToken(HEXADECIMAL_INFO, asciiString(start, 0)); |
| 430 return next; | 430 return next; |
| 431 } | 431 } |
| 432 } | 432 } |
| 433 } | 433 } |
| 434 | 434 |
| 435 int tokenizeDotOrNumber(int next) { | 435 int tokenizeDotOrNumber(int next) { |
| 436 int start = byteOffset; | 436 int start = byteOffset; |
| 437 next = advance(); | 437 next = advance(); |
| (...skipping 40 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 478 int tokenizeExponent(int next) { | 478 int tokenizeExponent(int next) { |
| 479 if (next === $PLUS || next === $MINUS) { | 479 if (next === $PLUS || next === $MINUS) { |
| 480 next = advance(); | 480 next = advance(); |
| 481 } | 481 } |
| 482 bool hasDigits = false; | 482 bool hasDigits = false; |
| 483 while (true) { | 483 while (true) { |
| 484 if ($0 <= next && next <= $9) { | 484 if ($0 <= next && next <= $9) { |
| 485 hasDigits = true; | 485 hasDigits = true; |
| 486 } else { | 486 } else { |
| 487 if (!hasDigits) { | 487 if (!hasDigits) { |
| 488 throw new MalformedInputException(charOffset); | 488 throw new MalformedInputException("digit expected", charOffset); |
| 489 } | 489 } |
| 490 return next; | 490 return next; |
| 491 } | 491 } |
| 492 next = advance(); | 492 next = advance(); |
| 493 } | 493 } |
| 494 } | 494 } |
| 495 | 495 |
| 496 int tokenizeSlashOrComment(int next) { | 496 int tokenizeSlashOrComment(int next) { |
| 497 next = advance(); | 497 next = advance(); |
| 498 if ($STAR === next) { | 498 if ($STAR === next) { |
| (...skipping 98 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 597 } | 597 } |
| 598 } | 598 } |
| 599 } | 599 } |
| 600 | 600 |
| 601 int tokenizeRawString(int next) { | 601 int tokenizeRawString(int next) { |
| 602 int start = byteOffset; | 602 int start = byteOffset; |
| 603 next = advance(); | 603 next = advance(); |
| 604 if (next === $DQ || next === $SQ) { | 604 if (next === $DQ || next === $SQ) { |
| 605 return tokenizeString(next, start, true); | 605 return tokenizeString(next, start, true); |
| 606 } else { | 606 } else { |
| 607 throw new MalformedInputException(charOffset); | 607 throw new MalformedInputException("expected ' or \"", charOffset); |
| 608 } | 608 } |
| 609 } | 609 } |
| 610 | 610 |
| 611 int tokenizeString(int next, int start, bool raw) { | 611 int tokenizeString(int next, int start, bool raw) { |
| 612 int q = next; | 612 int q = next; |
| 613 next = advance(); | 613 next = advance(); |
| 614 if (q === next) { | 614 if (q === next) { |
| 615 next = advance(); | 615 next = advance(); |
| 616 if (q === next) { | 616 if (q === next) { |
| 617 // Multiline string. | 617 // Multiline string. |
| (...skipping 18 matching lines...) Expand all Loading... |
| 636 } | 636 } |
| 637 | 637 |
| 638 int tokenizeSingleLineString(int next, int q1, int start) { | 638 int tokenizeSingleLineString(int next, int q1, int start) { |
| 639 while (next !== $EOF) { | 639 while (next !== $EOF) { |
| 640 if (next === q1) { | 640 if (next === q1) { |
| 641 appendByteStringToken(STRING_INFO, utf8String(start, 0)); | 641 appendByteStringToken(STRING_INFO, utf8String(start, 0)); |
| 642 return advance(); | 642 return advance(); |
| 643 } else if (next === $BACKSLASH) { | 643 } else if (next === $BACKSLASH) { |
| 644 next = advance(); | 644 next = advance(); |
| 645 if (next === $EOF) { | 645 if (next === $EOF) { |
| 646 throw new MalformedInputException(charOffset); | 646 throw new MalformedInputException("unterminated string literal", |
| 647 charOffset); |
| 647 } | 648 } |
| 648 } else if (next === $$) { | 649 } else if (next === $$) { |
| 649 beginToken(); | 650 beginToken(); |
| 650 next = advance(); | 651 next = advance(); |
| 651 if (next === $OPEN_CURLY_BRACKET) { | 652 if (next === $OPEN_CURLY_BRACKET) { |
| 652 next = tokenizeInterpolatedExpression(next, start); | 653 next = tokenizeInterpolatedExpression(next, start); |
| 653 } else { | 654 } else { |
| 654 next = tokenizeInterpolatedIdentifier(next, start); | 655 next = tokenizeInterpolatedIdentifier(next, start); |
| 655 } | 656 } |
| 656 start = byteOffset; | 657 start = byteOffset; |
| 657 continue; | 658 continue; |
| 658 } else if (next === $LF || next === $CR) { | 659 } else if (next === $LF || next === $CR) { |
| 659 throw new MalformedInputException(charOffset); | 660 throw new MalformedInputException("unterminated string literal", |
| 661 charOffset); |
| 660 } | 662 } |
| 661 next = advance(); | 663 next = advance(); |
| 662 } | 664 } |
| 663 throw new MalformedInputException(charOffset); | 665 throw new MalformedInputException("unterminated string literal", |
| 666 charOffset); |
| 664 } | 667 } |
| 665 | 668 |
| 666 int tokenizeInterpolatedExpression(int next, int start) { | 669 int tokenizeInterpolatedExpression(int next, int start) { |
| 667 appendByteStringToken(STRING_INFO, utf8String(start, -2)); | 670 appendByteStringToken(STRING_INFO, utf8String(start, -2)); |
| 668 appendBeginGroup(STRING_INTERPOLATION_INFO, "\${"); | 671 appendBeginGroup(STRING_INTERPOLATION_INFO, "\${"); |
| 669 next = advance(); | 672 next = advance(); |
| 670 while (next !== $EOF && next !== $STX) { | 673 while (next !== $EOF && next !== $STX) { |
| 671 next = bigSwitch(next); | 674 next = bigSwitch(next); |
| 672 } | 675 } |
| 673 if (next === $EOF) return next; | 676 if (next === $EOF) return next; |
| 674 return advance(); | 677 return advance(); |
| 675 } | 678 } |
| 676 | 679 |
| 677 int tokenizeInterpolatedIdentifier(int next, int start) { | 680 int tokenizeInterpolatedIdentifier(int next, int start) { |
| 678 appendByteStringToken(STRING_INFO, utf8String(start, -2)); | 681 appendByteStringToken(STRING_INFO, utf8String(start, -2)); |
| 679 appendBeginGroup(STRING_INTERPOLATION_INFO, "\${"); | 682 appendBeginGroup(STRING_INTERPOLATION_INFO, "\${"); |
| 680 next = tokenizeKeywordOrIdentifier(next, false); | 683 next = tokenizeKeywordOrIdentifier(next, false); |
| 681 appendEndGroup(CLOSE_CURLY_BRACKET_INFO, "}", OPEN_CURLY_BRACKET_TOKEN); | 684 appendEndGroup(CLOSE_CURLY_BRACKET_INFO, "}", OPEN_CURLY_BRACKET_TOKEN); |
| 682 return next; | 685 return next; |
| 683 } | 686 } |
| 684 | 687 |
| 685 int tokenizeSingleLineRawString(int next, int q1, int start) { | 688 int tokenizeSingleLineRawString(int next, int q1, int start) { |
| 686 next = advance(); | 689 next = advance(); |
| 687 while (next != $EOF) { | 690 while (next != $EOF) { |
| 688 if (next === q1) { | 691 if (next === q1) { |
| 689 appendByteStringToken(STRING_INFO, utf8String(start, 0)); | 692 appendByteStringToken(STRING_INFO, utf8String(start, 0)); |
| 690 return advance(); | 693 return advance(); |
| 691 } else if (next === $LF || next === $CR) { | 694 } else if (next === $LF || next === $CR) { |
| 692 throw new MalformedInputException(charOffset); | 695 throw new MalformedInputException("unterminated string literal", |
| 696 charOffset); |
| 693 } | 697 } |
| 694 next = advance(); | 698 next = advance(); |
| 695 } | 699 } |
| 696 throw new MalformedInputException(charOffset); | 700 throw new MalformedInputException("unterminated string literal", |
| 701 charOffset); |
| 697 } | 702 } |
| 698 | 703 |
| 699 int tokenizeMultiLineString(int q, int start, bool raw) { | 704 int tokenizeMultiLineString(int q, int start, bool raw) { |
| 700 // TODO(ahe): Handle escapes. | 705 // TODO(ahe): Handle escapes. |
| 701 // TODO(ahe): Handle string interpolation. | 706 // TODO(ahe): Handle string interpolation. |
| 702 int next = advance(); | 707 int next = advance(); |
| 703 while (next != $EOF) { | 708 while (next != $EOF) { |
| 704 if (next === q) { | 709 if (next === q) { |
| 705 next = advance(); | 710 next = advance(); |
| 706 if (next === q) { | 711 if (next === q) { |
| 707 next = advance(); | 712 next = advance(); |
| 708 if (next === q) { | 713 if (next === q) { |
| 709 appendByteStringToken(STRING_INFO, utf8String(start, 0)); | 714 appendByteStringToken(STRING_INFO, utf8String(start, 0)); |
| 710 return advance(); | 715 return advance(); |
| 711 } | 716 } |
| 712 } | 717 } |
| 713 } | 718 } |
| 714 next = advance(); | 719 next = advance(); |
| 715 } | 720 } |
| 716 return next; | 721 return next; |
| 717 } | 722 } |
| 718 } | 723 } |
| 719 | 724 |
| 720 class MalformedInputException { | 725 class MalformedInputException { |
| 721 final message; | 726 final String message; |
| 722 MalformedInputException(this.message); | 727 final position; |
| 723 toString() => message.toString(); | 728 MalformedInputException(this.message, this.position); |
| 729 toString() => message; |
| 724 } | 730 } |
| OLD | NEW |