| OLD | NEW |
| 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 interface Scanner { | 5 interface Scanner { |
| 6 Token tokenize(); | 6 Token tokenize(); |
| 7 } | 7 } |
| 8 | 8 |
| 9 /** | 9 /** |
| 10 * Common base class for a Dart scanner. | 10 * Common base class for a Dart scanner. |
| (...skipping 635 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 646 int tokenizeIdentifier(int next, int start, bool allowDollar) { | 646 int tokenizeIdentifier(int next, int start, bool allowDollar) { |
| 647 bool isAscii = true; | 647 bool isAscii = true; |
| 648 while (true) { | 648 while (true) { |
| 649 if (($a <= next && next <= $z) || | 649 if (($a <= next && next <= $z) || |
| 650 ($A <= next && next <= $Z) || | 650 ($A <= next && next <= $Z) || |
| 651 ($0 <= next && next <= $9) || | 651 ($0 <= next && next <= $9) || |
| 652 next === $_ || | 652 next === $_ || |
| 653 (next === $$ && allowDollar)) { | 653 (next === $$ && allowDollar)) { |
| 654 next = advance(); | 654 next = advance(); |
| 655 } else if (next < 128) { | 655 } else if (next < 128) { |
| 656 // Identifier ends here. |
| 657 if (start == byteOffset) { |
| 658 throw new MalformedInputException("expected identifier not found", |
| 659 charOffset); |
| 660 } |
| 656 if (isAscii) { | 661 if (isAscii) { |
| 657 appendByteStringToken(IDENTIFIER_INFO, asciiString(start, 0)); | 662 appendByteStringToken(IDENTIFIER_INFO, asciiString(start, 0)); |
| 658 } else { | 663 } else { |
| 659 appendByteStringToken(IDENTIFIER_INFO, utf8String(start, -1)); | 664 appendByteStringToken(IDENTIFIER_INFO, utf8String(start, -1)); |
| 660 } | 665 } |
| 661 return next; | 666 return next; |
| 662 } else { | 667 } else { |
| 663 int nonAsciiStart = byteOffset; | 668 int nonAsciiStart = byteOffset; |
| 664 do { | 669 do { |
| 665 next = nextByte(); | 670 next = nextByte(); |
| (...skipping 161 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 827 charOffset); | 832 charOffset); |
| 828 } | 833 } |
| 829 } | 834 } |
| 830 | 835 |
| 831 class MalformedInputException { | 836 class MalformedInputException { |
| 832 final String message; | 837 final String message; |
| 833 final position; | 838 final position; |
| 834 MalformedInputException(this.message, this.position); | 839 MalformedInputException(this.message, this.position); |
| 835 toString() => message; | 840 toString() => message; |
| 836 } | 841 } |
| OLD | NEW |