OLD | NEW |
1 // Copyright 2011 the V8 project authors. All rights reserved. | 1 // Copyright 2011 the V8 project authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
4 | 4 |
5 // Features shared by parsing and pre-parsing scanners. | 5 // Features shared by parsing and pre-parsing scanners. |
6 | 6 |
7 #include <stdint.h> | 7 #include <stdint.h> |
8 | 8 |
9 #include <cmath> | 9 #include <cmath> |
10 | 10 |
(...skipping 20 matching lines...) Expand all Loading... |
31 | 31 |
32 // ---------------------------------------------------------------------------- | 32 // ---------------------------------------------------------------------------- |
33 // Scanner | 33 // Scanner |
34 | 34 |
35 Scanner::Scanner(UnicodeCache* unicode_cache) | 35 Scanner::Scanner(UnicodeCache* unicode_cache) |
36 : unicode_cache_(unicode_cache), | 36 : unicode_cache_(unicode_cache), |
37 octal_pos_(Location::invalid()), | 37 octal_pos_(Location::invalid()), |
38 harmony_scoping_(false), | 38 harmony_scoping_(false), |
39 harmony_modules_(false), | 39 harmony_modules_(false), |
40 harmony_numeric_literals_(false), | 40 harmony_numeric_literals_(false), |
41 harmony_classes_(false) { } | 41 harmony_classes_(false), |
| 42 harmony_templates_(false) {} |
42 | 43 |
43 | 44 |
44 void Scanner::Initialize(Utf16CharacterStream* source) { | 45 void Scanner::Initialize(Utf16CharacterStream* source) { |
45 source_ = source; | 46 source_ = source; |
46 // Need to capture identifiers in order to recognize "get" and "set" | 47 // Need to capture identifiers in order to recognize "get" and "set" |
47 // in object literals. | 48 // in object literals. |
48 Init(); | 49 Init(); |
49 // Skip initial whitespace allowing HTML comment ends just like | 50 // Skip initial whitespace allowing HTML comment ends just like |
50 // after a newline and scan first token. | 51 // after a newline and scan first token. |
51 has_line_terminator_before_next_ = true; | 52 has_line_terminator_before_next_ = true; |
(...skipping 567 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
619 break; | 620 break; |
620 | 621 |
621 case '?': | 622 case '?': |
622 token = Select(Token::CONDITIONAL); | 623 token = Select(Token::CONDITIONAL); |
623 break; | 624 break; |
624 | 625 |
625 case '~': | 626 case '~': |
626 token = Select(Token::BIT_NOT); | 627 token = Select(Token::BIT_NOT); |
627 break; | 628 break; |
628 | 629 |
| 630 case '`': |
| 631 if (HarmonyTemplates()) { |
| 632 token = ScanTemplateSpan(); |
| 633 break; |
| 634 } |
| 635 |
629 default: | 636 default: |
630 if (c0_ < 0) { | 637 if (c0_ < 0) { |
631 token = Token::EOS; | 638 token = Token::EOS; |
632 } else if (unicode_cache_->IsIdentifierStart(c0_)) { | 639 } else if (unicode_cache_->IsIdentifierStart(c0_)) { |
633 token = ScanIdentifierOrKeyword(); | 640 token = ScanIdentifierOrKeyword(); |
634 } else if (IsDecimalDigit(c0_)) { | 641 } else if (IsDecimalDigit(c0_)) { |
635 token = ScanNumber(false); | 642 token = ScanNumber(false); |
636 } else if (SkipWhiteSpace()) { | 643 } else if (SkipWhiteSpace()) { |
637 token = Token::WHITESPACE; | 644 token = Token::WHITESPACE; |
638 } else { | 645 } else { |
(...skipping 124 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
763 } | 770 } |
764 } | 771 } |
765 if (c0_ != quote) return Token::ILLEGAL; | 772 if (c0_ != quote) return Token::ILLEGAL; |
766 literal.Complete(); | 773 literal.Complete(); |
767 | 774 |
768 Advance(); // consume quote | 775 Advance(); // consume quote |
769 return Token::STRING; | 776 return Token::STRING; |
770 } | 777 } |
771 | 778 |
772 | 779 |
| 780 Token::Value Scanner::ScanTemplateSpan() { |
| 781 // When scanning a TemplateSpan, we are looking for the following construct: |
| 782 // TEMPLATE_SPAN :: |
| 783 // ` LiteralChars* ${ |
| 784 // | } LiteralChars* ${ |
| 785 // |
| 786 // TEMPLATE_TAIL :: |
| 787 // ` LiteralChars* ` |
| 788 // | } LiteralChar* ` |
| 789 // |
| 790 // A TEMPLATE_SPAN should always be followed by an Expression, while a |
| 791 // TEMPLATE_TAIL terminates a TemplateLiteral and does not need to be |
| 792 // followed by an Expression. |
| 793 // |
| 794 |
| 795 if (next_.token == Token::RBRACE) { |
| 796 // After parsing an Expression, the source position is incorrect due to |
| 797 // having scanned the brace. Push the RBRACE back into the stream. |
| 798 PushBack('}'); |
| 799 } |
| 800 |
| 801 next_.location.beg_pos = source_pos(); |
| 802 Token::Value result = Token::TEMPLATE_SPAN; |
| 803 DCHECK(c0_ == '`' || c0_ == '}'); |
| 804 Advance(); // Consume ` or } |
| 805 |
| 806 LiteralScope literal(this); |
| 807 while (true) { |
| 808 uc32 c = c0_; |
| 809 Advance(); |
| 810 if (c == '`') { |
| 811 result = Token::TEMPLATE_TAIL; |
| 812 break; |
| 813 } else if (c == '$' && c0_ == '{') { |
| 814 Advance(); // Consume '{' |
| 815 break; |
| 816 } else if (c == '\\') { |
| 817 if (unicode_cache_->IsLineTerminator(c0_)) { |
| 818 // The TV of LineContinuation :: \ LineTerminatorSequence is the empty |
| 819 // code unit sequence. |
| 820 uc32 lastChar = c0_; |
| 821 Advance(); |
| 822 if (lastChar == '\r' && c0_ == '\n') Advance(); |
| 823 } else if (c0_ == '0') { |
| 824 Advance(); |
| 825 AddLiteralChar('0'); |
| 826 } else { |
| 827 ScanEscape(); |
| 828 } |
| 829 } else if (c < 0) { |
| 830 // Unterminated template literal |
| 831 PushBack(c); |
| 832 break; |
| 833 } else { |
| 834 // The TRV of LineTerminatorSequence :: <CR> is the CV 0x000A. |
| 835 // The TRV of LineTerminatorSequence :: <CR><LF> is the sequence |
| 836 // consisting of the CV 0x000A. |
| 837 if (c == '\r') { |
| 838 if (c0_ == '\n') Advance(); |
| 839 c = '\n'; |
| 840 } |
| 841 AddLiteralChar(c); |
| 842 } |
| 843 } |
| 844 literal.Complete(); |
| 845 next_.location.end_pos = source_pos(); |
| 846 next_.token = result; |
| 847 return result; |
| 848 } |
| 849 |
| 850 |
773 void Scanner::ScanDecimalDigits() { | 851 void Scanner::ScanDecimalDigits() { |
774 while (IsDecimalDigit(c0_)) | 852 while (IsDecimalDigit(c0_)) |
775 AddLiteralCharAdvance(); | 853 AddLiteralCharAdvance(); |
776 } | 854 } |
777 | 855 |
778 | 856 |
779 Token::Value Scanner::ScanNumber(bool seen_period) { | 857 Token::Value Scanner::ScanNumber(bool seen_period) { |
780 DCHECK(IsDecimalDigit(c0_)); // the first digit of the number or the fraction | 858 DCHECK(IsDecimalDigit(c0_)); // the first digit of the number or the fraction |
781 | 859 |
782 enum { DECIMAL, HEX, OCTAL, IMPLICIT_OCTAL, BINARY } kind = DECIMAL; | 860 enum { DECIMAL, HEX, OCTAL, IMPLICIT_OCTAL, BINARY } kind = DECIMAL; |
(...skipping 543 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1326 } | 1404 } |
1327 backing_store_.Add(static_cast<uint8_t>((one_byte_length >> 7) | 0x80u)); | 1405 backing_store_.Add(static_cast<uint8_t>((one_byte_length >> 7) | 0x80u)); |
1328 } | 1406 } |
1329 backing_store_.Add(static_cast<uint8_t>(one_byte_length & 0x7f)); | 1407 backing_store_.Add(static_cast<uint8_t>(one_byte_length & 0x7f)); |
1330 | 1408 |
1331 backing_store_.AddBlock(bytes); | 1409 backing_store_.AddBlock(bytes); |
1332 return backing_store_.EndSequence().start(); | 1410 return backing_store_.EndSequence().start(); |
1333 } | 1411 } |
1334 | 1412 |
1335 } } // namespace v8::internal | 1413 } } // namespace v8::internal |
OLD | NEW |