| OLD | NEW |
| 1 // Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 #import('../../../../lib/utf8/utf8.dart'); | 5 #import('../../../../lib/utf8/utf8.dart'); |
| 6 #import('../../../leg/scanner/scannerlib.dart'); | 6 #import('../../../leg/scanner/scannerlib.dart'); |
| 7 #import('../../../leg/scanner/scanner_implementation.dart'); | 7 #import('../../../leg/scanner/scanner_implementation.dart'); |
| 8 #import('../../../leg/util/characters.dart'); | 8 #import('../../../leg/util/characters.dart'); |
| 9 #source('../../../leg/scanner/byte_strings.dart'); | 9 #source('../../../leg/scanner/byte_strings.dart'); |
| 10 #source('../../../leg/scanner/byte_array_scanner.dart'); | 10 #source('../../../leg/scanner/byte_array_scanner.dart'); |
| 11 | 11 |
| 12 Token scan(List<int> bytes) => new ByteArrayScanner(bytes).tokenize(); | 12 Token scan(List<int> bytes) => new ByteArrayScanner(bytes).tokenize(); |
| 13 | 13 |
| 14 bool isRunningOnJavaScript() => 1 === 1.0; | 14 bool isRunningOnJavaScript() => 1 === 1.0; |
| 15 | 15 |
| 16 main() { | 16 main() { |
| 17 // Google favorite: "Îñţérñåţîöñåļîžåţîờñ". | 17 // Google favorite: "Îñţérñåţîöñåļîžåţîờñ". |
| 18 Token token = scan([0xc3, 0x8e, 0xc3, 0xb1, 0xc5, 0xa3, 0xc3, 0xa9, 0x72, | 18 Token token = scan([0xc3, 0x8e, 0xc3, 0xb1, 0xc5, 0xa3, 0xc3, 0xa9, 0x72, |
| 19 0xc3, 0xb1, 0xc3, 0xa5, 0xc5, 0xa3, 0xc3, 0xae, 0xc3, | 19 0xc3, 0xb1, 0xc3, 0xa5, 0xc5, 0xa3, 0xc3, 0xae, 0xc3, |
| 20 0xb6, 0xc3, 0xb1, 0xc3, 0xa5, 0xc4, 0xbc, 0xc3, 0xae, | 20 0xb6, 0xc3, 0xb1, 0xc3, 0xa5, 0xc4, 0xbc, 0xc3, 0xae, |
| 21 0xc5, 0xbe, 0xc3, 0xa5, 0xc5, 0xa3, 0xc3, 0xae, 0xe1, | 21 0xc5, 0xbe, 0xc3, 0xa5, 0xc5, 0xa3, 0xc3, 0xae, 0xe1, |
| 22 0xbb, 0x9d, 0xc3, 0xb1, $EOF]); | 22 0xbb, 0x9d, 0xc3, 0xb1, $EOF]); |
| 23 Expect.stringEquals("Îñţérñåţîöñåļîžåţîờñ", token.value.toString()); | 23 Expect.stringEquals("Îñţérñåţîöñåļîžåţîờñ", token.value.slowToString()); |
| 24 | 24 |
| 25 // Blueberry porridge in Danish: "blåbærgrød". | 25 // Blueberry porridge in Danish: "blåbærgrød". |
| 26 token = scan([0x62, 0x6c, 0xc3, 0xa5, 0x62, 0xc3, 0xa6, 0x72, 0x67, 0x72, | 26 token = scan([0x62, 0x6c, 0xc3, 0xa5, 0x62, 0xc3, 0xa6, 0x72, 0x67, 0x72, |
| 27 0xc3, 0xb8, 0x64, $EOF]); | 27 0xc3, 0xb8, 0x64, $EOF]); |
| 28 Expect.stringEquals("blåbærgrød", token.value.toString()); | 28 Expect.stringEquals("blåbærgrød", token.value.slowToString()); |
| 29 | 29 |
| 30 // "சிவா அணாமாைல", that is "Siva Annamalai" in Tamil. | 30 // "சிவா அணாமாைல", that is "Siva Annamalai" in Tamil. |
| 31 token = scan([0xe0, 0xae, 0x9a, 0xe0, 0xae, 0xbf, 0xe0, 0xae, 0xb5, 0xe0, | 31 token = scan([0xe0, 0xae, 0x9a, 0xe0, 0xae, 0xbf, 0xe0, 0xae, 0xb5, 0xe0, |
| 32 0xae, 0xbe, 0x20, 0xe0, 0xae, 0x85, 0xe0, 0xae, 0xa3, 0xe0, | 32 0xae, 0xbe, 0x20, 0xe0, 0xae, 0x85, 0xe0, 0xae, 0xa3, 0xe0, |
| 33 0xae, 0xbe, 0xe0, 0xae, 0xae, 0xe0, 0xae, 0xbe, 0xe0, 0xaf, | 33 0xae, 0xbe, 0xe0, 0xae, 0xae, 0xe0, 0xae, 0xbe, 0xe0, 0xaf, |
| 34 0x88, 0xe0, 0xae, 0xb2, $EOF]); | 34 0x88, 0xe0, 0xae, 0xb2, $EOF]); |
| 35 Expect.stringEquals("சிவா", token.value.toString()); | 35 Expect.stringEquals("சிவா", token.value.slowToString()); |
| 36 Expect.stringEquals("அணாமாைல", token.next.toString()); | 36 Expect.stringEquals("அணாமாைல", token.next.value.slowToString()); |
| 37 | 37 |
| 38 // "िसवा अणामालै", that is "Siva Annamalai" in Devanagari. | 38 // "िसवा अणामालै", that is "Siva Annamalai" in Devanagari. |
| 39 token = scan([0xe0, 0xa4, 0xbf, 0xe0, 0xa4, 0xb8, 0xe0, 0xa4, 0xb5, 0xe0, | 39 token = scan([0xe0, 0xa4, 0xbf, 0xe0, 0xa4, 0xb8, 0xe0, 0xa4, 0xb5, 0xe0, |
| 40 0xa4, 0xbe, 0x20, 0xe0, 0xa4, 0x85, 0xe0, 0xa4, 0xa3, 0xe0, | 40 0xa4, 0xbe, 0x20, 0xe0, 0xa4, 0x85, 0xe0, 0xa4, 0xa3, 0xe0, |
| 41 0xa4, 0xbe, 0xe0, 0xa4, 0xae, 0xe0, 0xa4, 0xbe, 0xe0, 0xa4, | 41 0xa4, 0xbe, 0xe0, 0xa4, 0xae, 0xe0, 0xa4, 0xbe, 0xe0, 0xa4, |
| 42 0xb2, 0xe0, 0xa5, 0x88, $EOF]); | 42 0xb2, 0xe0, 0xa5, 0x88, $EOF]); |
| 43 Expect.stringEquals("िसवा", token.value.toString()); | 43 Expect.stringEquals("िसवा", token.value.slowToString()); |
| 44 Expect.stringEquals("अणामालै", token.next.toString()); | 44 Expect.stringEquals("अणामालै", token.next.value.slowToString()); |
| 45 | 45 |
| 46 if (!isRunningOnJavaScript()) { | 46 if (!isRunningOnJavaScript()) { |
| 47 // DESERET CAPITAL LETTER BEE, unicode 0x10412(0xD801+0xDC12) | 47 // DESERET CAPITAL LETTER BEE, unicode 0x10412(0xD801+0xDC12) |
| 48 // UTF-8: F0 90 90 92 | 48 // UTF-8: F0 90 90 92 |
| 49 token = scan([0xf0, 0x90, 0x90, 0x92, $EOF]); | 49 token = scan([0xf0, 0x90, 0x90, 0x92, $EOF]); |
| 50 Expect.stringEquals("𐐒", token.value.toString()); | 50 Expect.stringEquals("𐐒", token.value.slowToString()); |
| 51 } else { | 51 } else { |
| 52 print('Skipping non-BMP character test'); | 52 print('Skipping non-BMP character test'); |
| 53 } | 53 } |
| 54 } | 54 } |
| OLD | NEW |