OLD | NEW |
| (Empty) |
1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | |
2 // for details. All rights reserved. Use of this source code is governed by a | |
3 // BSD-style license that can be found in the LICENSE file. | |
4 | |
5 #import('../../../lib/utf/utf.dart'); | |
6 #import('../../../lib/compiler/implementation/scanner/scannerlib.dart'); | |
7 #import('../../../lib/compiler/implementation/scanner/scanner_implementation.dar
t'); | |
8 #import('../../../lib/compiler/implementation/util/characters.dart'); | |
9 #source('../../../lib/compiler/implementation/scanner/byte_strings.dart'); | |
10 #source('../../../lib/compiler/implementation/scanner/byte_array_scanner.dart'); | |
11 | |
12 Token scan(List<int> bytes) => new ByteArrayScanner(bytes).tokenize(); | |
13 | |
14 bool isRunningOnJavaScript() => 1 === 1.0; | |
15 | |
16 main() { | |
17 // Google favorite: "Îñţérñåţîöñåļîžåţîờñ". | |
18 Token token = scan([0xc3, 0x8e, 0xc3, 0xb1, 0xc5, 0xa3, 0xc3, 0xa9, 0x72, | |
19 0xc3, 0xb1, 0xc3, 0xa5, 0xc5, 0xa3, 0xc3, 0xae, 0xc3, | |
20 0xb6, 0xc3, 0xb1, 0xc3, 0xa5, 0xc4, 0xbc, 0xc3, 0xae, | |
21 0xc5, 0xbe, 0xc3, 0xa5, 0xc5, 0xa3, 0xc3, 0xae, 0xe1, | |
22 0xbb, 0x9d, 0xc3, 0xb1, $EOF]); | |
23 Expect.stringEquals("Îñţérñåţîöñåļîžåţîờñ", token.value.slowToString()); | |
24 | |
25 // Blueberry porridge in Danish: "blåbærgrød". | |
26 token = scan([0x62, 0x6c, 0xc3, 0xa5, 0x62, 0xc3, 0xa6, 0x72, 0x67, 0x72, | |
27 0xc3, 0xb8, 0x64, $EOF]); | |
28 Expect.stringEquals("blåbærgrød", token.value.slowToString()); | |
29 | |
30 // "சிவா அணாமாைல", that is "Siva Annamalai" in Tamil. | |
31 token = scan([0xe0, 0xae, 0x9a, 0xe0, 0xae, 0xbf, 0xe0, 0xae, 0xb5, 0xe0, | |
32 0xae, 0xbe, 0x20, 0xe0, 0xae, 0x85, 0xe0, 0xae, 0xa3, 0xe0, | |
33 0xae, 0xbe, 0xe0, 0xae, 0xae, 0xe0, 0xae, 0xbe, 0xe0, 0xaf, | |
34 0x88, 0xe0, 0xae, 0xb2, $EOF]); | |
35 Expect.stringEquals("சிவா", token.value.slowToString()); | |
36 Expect.stringEquals("அணாமாைல", token.next.value.slowToString()); | |
37 | |
38 // "िसवा अणामालै", that is "Siva Annamalai" in Devanagari. | |
39 token = scan([0xe0, 0xa4, 0xbf, 0xe0, 0xa4, 0xb8, 0xe0, 0xa4, 0xb5, 0xe0, | |
40 0xa4, 0xbe, 0x20, 0xe0, 0xa4, 0x85, 0xe0, 0xa4, 0xa3, 0xe0, | |
41 0xa4, 0xbe, 0xe0, 0xa4, 0xae, 0xe0, 0xa4, 0xbe, 0xe0, 0xa4, | |
42 0xb2, 0xe0, 0xa5, 0x88, $EOF]); | |
43 Expect.stringEquals("िसवा", token.value.slowToString()); | |
44 Expect.stringEquals("अणामालै", token.next.value.slowToString()); | |
45 | |
46 if (!isRunningOnJavaScript()) { | |
47 // DESERET CAPITAL LETTER BEE, unicode 0x10412(0xD801+0xDC12) | |
48 // UTF-8: F0 90 90 92 | |
49 token = scan([0xf0, 0x90, 0x90, 0x92, $EOF]); | |
50 Expect.stringEquals("𐐒", token.value.slowToString()); | |
51 } else { | |
52 print('Skipping non-BMP character test'); | |
53 } | |
54 | |
55 // Regression test for issue 1761. | |
56 // "#!" | |
57 token = scan([0x23, 0x21, $EOF]); | |
58 Expect.equals(token.info, EOF_INFO); // Treated as a comment. | |
59 | |
60 // Regression test for issue 1761. | |
61 // "#! Hello, World!" | |
62 token = scan([0x23, 0x21, 0x20, 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x2c, 0x20, | |
63 0x57, 0x6f, 0x72, 0x6c, 0x64, 0x21, $EOF]); | |
64 Expect.equals(token.info, EOF_INFO); // Treated as a comment. | |
65 } | |
OLD | NEW |