| OLD | NEW |
| 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 interface Scanner { | 5 interface Scanner { |
| 6 Token tokenize(); | 6 Token tokenize(); |
| 7 } | 7 } |
| 8 | 8 |
| 9 /** | 9 /** |
| 10 * Common base class for a Dart scanner. | 10 * Common base class for a Dart scanner. |
| (...skipping 571 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 582 appendByteStringToken(IDENTIFIER_INFO, asciiString(start, 0)); | 582 appendByteStringToken(IDENTIFIER_INFO, asciiString(start, 0)); |
| 583 } else { | 583 } else { |
| 584 appendByteStringToken(IDENTIFIER_INFO, utf8String(start, -1)); | 584 appendByteStringToken(IDENTIFIER_INFO, utf8String(start, -1)); |
| 585 } | 585 } |
| 586 return next; | 586 return next; |
| 587 } else { | 587 } else { |
| 588 int nonAsciiStart = byteOffset; | 588 int nonAsciiStart = byteOffset; |
| 589 do { | 589 do { |
| 590 next = nextByte(); | 590 next = nextByte(); |
| 591 } while (next > 127); | 591 } while (next > 127); |
| 592 String string = utf8String(nonAsciiStart, -1).toString(); | 592 String string = utf8String(nonAsciiStart, -1).slowToString(); |
| 593 isAscii = false; | 593 isAscii = false; |
| 594 int byteLength = nonAsciiStart - byteOffset; | 594 int byteLength = nonAsciiStart - byteOffset; |
| 595 addToCharOffset(string.length - byteLength); | 595 addToCharOffset(string.length - byteLength); |
| 596 } | 596 } |
| 597 } | 597 } |
| 598 } | 598 } |
| 599 | 599 |
| 600 int tokenizeRawString(int next) { | 600 int tokenizeRawString(int next) { |
| 601 int start = byteOffset; | 601 int start = byteOffset; |
| 602 next = advance(); | 602 next = advance(); |
| (...skipping 145 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 748 charOffset); | 748 charOffset); |
| 749 } | 749 } |
| 750 } | 750 } |
| 751 | 751 |
| 752 class MalformedInputException { | 752 class MalformedInputException { |
| 753 final String message; | 753 final String message; |
| 754 final position; | 754 final position; |
| 755 MalformedInputException(this.message, this.position); | 755 MalformedInputException(this.message, this.position); |
| 756 toString() => message; | 756 toString() => message; |
| 757 } | 757 } |
| OLD | NEW |