Chromium Code Reviews Chromium Code Reviews
Issue 9233041:
String encoding utility methods and tests for Unicode, UTF-8, -16 and -32. (Closed)
Base URL: https://dart.googlecode.com/svn/branches/bleeding_edge/dart| OLD | NEW |
|---|---|
| (Empty) | |
| 1 #!/usr/bin/env dart | |
| 2 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | |
| 3 // for details. All rights reserved. Use of this source code is governed by a | |
| 4 // BSD-style license that can be found in the LICENSE file. | |
| 5 | |
| 6 #library("Utf8Tests"); | |
| 7 #import("DUnit.dart"); | |
| 8 #import("../../string_encoding/Unicode.dart"); | |
| 9 #import("../../string_encoding/UnicodeCore.dart"); | |
| 10 #source("../../string_encoding/Utf8_impl.dart"); | |
| 11 | |
| 12 void main() { | |
| 13 TestSuite suite = new TestSuite(); | |
| 14 suite.registerTestClass(new Utf8Tests()); | |
| 15 suite.run(); | |
| 16 } | |
| 17 | |
| 18 class Utf8Tests extends TestClass { | |
| 19 void registerTests(TestSuite suite) { | |
| 20 register("testUtf8bytesToCodepoints", testUtf8bytesToCodepoints, suite); | |
| 21 register("testUtf8BytesToString", testUtf8BytesToString, suite); | |
| 22 register("testEncodeToUtf8", testEncodeToUtf8, suite); | |
| 23 } | |
| 24 | |
| 25 void testEncodeToUtf8() { | |
| 26 Expect.listEquals([ | |
| 27 0x54, 0x68, 0x65, 0x20, 0x71, 0x75, 0x69, 0x63, | |
| 28 0x6b, 0x20, 0x62, 0x72, 0x6f, 0x77, 0x6e, 0x20, | |
| 29 0x66, 0x6f, 0x78, 0x20, 0x6a, 0x75, 0x6d, 0x70, | |
| 30 0x73, 0x20, 0x6f, 0x76, 0x65, 0x72, 0x20, 0x74, | |
| 31 0x68, 0x65, 0x20, 0x6c, 0x61, 0x7a, 0x79, 0x20, | |
| 32 0x64, 0x6f, 0x67, 0x2e], | |
| 33 encodeAsUtf8("The quick brown fox jumps over the lazy dog."), | |
| 34 "english to utf8"); | |
| 35 | |
| 36 Expect.listEquals([ | |
| 37 0x51, 0x75, 0x69, 0x7a, 0x64, 0x65, 0x6c, 0x74, | |
| 38 0x61, 0x67, 0x65, 0x72, 0x6e, 0x65, 0x20, 0x73, | |
| 39 0x70, 0x69, 0x73, 0x74, 0x65, 0x20, 0x6a, 0x6f, | |
| 40 0x72, 0x64, 0x62, 0xc3, 0xa6, 0x72, 0x20, 0x6d, | |
| 41 0x65, 0x64, 0x20, 0x66, 0x6c, 0xc3, 0xb8, 0x64, | |
| 42 0x65, 0x20, 0x6d, 0x65, 0x6e, 0x73, 0x20, 0x63, | |
| 43 0x69, 0x72, 0x6b, 0x75, 0x73, 0x6b, 0x6c, 0x6f, | |
| 44 0x76, 0x6e, 0x65, 0x6e, 0x20, 0x57, 0x6f, 0x6c, | |
| 45 0x74, 0x68, 0x65, 0x72, 0x20, 0x73, 0x70, 0x69, | |
| 46 0x6c, 0x6c, 0x65, 0x64, 0x65, 0x20, 0x70, 0xc3, | |
| 47 0xa5, 0x20, 0x78, 0x79, 0x6c, 0x6f, 0x66, 0x6f, | |
| 48 0x6e, 0x2e], | |
| 49 encodeAsUtf8("Quizdeltagerne spiste jordbær med fløde mens " + | |
| 50 "cirkusklovnen Wolther spillede på xylofon."), | |
| 51 "encode danish to utf8"); | |
| 52 | |
| 53 Expect.listEquals([ | |
| 54 0xd7, 0x93, 0xd7, 0x92, 0x20, 0xd7, 0xa1, 0xd7, | |
| 55 0xa7, 0xd7, 0xa8, 0xd7, 0x9f, 0x20, 0xd7, 0xa9, | |
| 56 0xd7, 0x98, 0x20, 0xd7, 0x91, 0xd7, 0x99, 0xd7, | |
| 57 0x9d, 0x20, 0xd7, 0x9e, 0xd7, 0x90, 0xd7, 0x95, | |
| 58 0xd7, 0x9b, 0xd7, 0x96, 0xd7, 0x91, 0x20, 0xd7, | |
| 59 0x95, 0xd7, 0x9c, 0xd7, 0xa4, 0xd7, 0xaa, 0xd7, | |
| 60 0xa2, 0x20, 0xd7, 0x9e, 0xd7, 0xa6, 0xd7, 0x90, | |
| 61 0x20, 0xd7, 0x9c, 0xd7, 0x95, 0x20, 0xd7, 0x97, | |
| 62 0xd7, 0x91, 0xd7, 0xa8, 0xd7, 0x94, 0x20, 0xd7, | |
| 63 0x90, 0xd7, 0x99, 0xd7, 0x9a, 0x20, 0xd7, 0x94, | |
| 64 0xd7, 0xa7, 0xd7, 0x9c, 0xd7, 0x99, 0xd7, 0x98, | |
| 65 0xd7, 0x94], | |
| 66 encodeAsUtf8("דג סקרן שט בים מאוכזב ולפתע מצא לו חברה איך הקליטה"), | |
| 67 "Hebrew to utf8"); | |
| 68 | |
| 69 Expect.listEquals([ | |
| 70 0xd0, 0xa1, 0xd1, 0x8a, 0xd0, 0xb5, 0xd1, 0x88, | |
| 71 0xd1, 0x8c, 0x20, 0xd0, 0xb6, 0xd0, 0xb5, 0x20, | |
| 72 0xd0, 0xb5, 0xd1, 0x89, 0xd1, 0x91, 0x20, 0xd1, | |
| 73 0x8d, 0xd1, 0x82, 0xd0, 0xb8, 0xd1, 0x85, 0x20, | |
| 74 0xd0, 0xbc, 0xd1, 0x8f, 0xd0, 0xb3, 0xd0, 0xba, | |
| 75 0xd0, 0xb8, 0xd1, 0x85, 0x20, 0xd1, 0x84, 0xd1, | |
| 76 0x80, 0xd0, 0xb0, 0xd0, 0xbd, 0xd1, 0x86, 0xd1, | |
| 77 0x83, 0xd0, 0xb7, 0xd1, 0x81, 0xd0, 0xba, 0xd0, | |
| 78 0xb8, 0xd1, 0x85, 0x20, 0xd0, 0xb1, 0xd1, 0x83, | |
| 79 0xd0, 0xbb, 0xd0, 0xbe, 0xd0, 0xba, 0x20, 0xd0, | |
| 80 0xb4, 0xd0, 0xb0, 0x20, 0xd0, 0xb2, 0xd1, 0x8b, | |
| 81 0xd0, 0xbf, 0xd0, 0xb5, 0xd0, 0xb9, 0x20, 0xd1, | |
| 82 0x87, 0xd0, 0xb0, 0xd1, 0x8e], | |
| 83 encodeAsUtf8("Съешь же ещё этих мягких французских булок да выпей" + | |
| 84 " чаю"), "Russian #2 to utf8"); | |
| 85 | |
| 86 Expect.listEquals([ | |
| 87 0xce, 0x93, 0xce, 0xb1, 0xce, 0xb6, 0xce, 0xad, | |
| 88 0xce, 0xb5, 0xcf, 0x82, 0x20, 0xce, 0xba, 0xce, | |
| 89 0xb1, 0xe1, 0xbd, 0xb6, 0x20, 0xce, 0xbc, 0xcf, | |
| 90 0x85, 0xcf, 0x81, 0xcf, 0x84, 0xce, 0xb9, 0xe1, | |
| 91 0xbd, 0xb2, 0xcf, 0x82, 0x20, 0xce, 0xb4, 0xe1, | |
| 92 0xbd, 0xb2, 0xce, 0xbd, 0x20, 0xce, 0xb8, 0xe1, | |
| 93 0xbd, 0xb0, 0x20, 0xce, 0xb2, 0xcf, 0x81, 0xe1, | |
| 94 0xbf, 0xb6, 0x20, 0xcf, 0x80, 0xce, 0xb9, 0xe1, | |
| 95 0xbd, 0xb0, 0x20, 0xcf, 0x83, 0xcf, 0x84, 0xe1, | |
| 96 0xbd, 0xb8, 0x20, 0xcf, 0x87, 0xcf, 0x81, 0xcf, | |
| 97 0x85, 0xcf, 0x83, 0xce, 0xb1, 0xcf, 0x86, 0xe1, | |
| 98 0xbd, 0xb6, 0x20, 0xce, 0xbe, 0xce, 0xad, 0xcf, | |
| 99 0x86, 0xcf, 0x89, 0xcf, 0x84, 0xce, 0xbf], | |
| 100 encodeAsUtf8("Γαζέες καὶ μυρτιὲς δὲν θὰ βρῶ πιὰ στὸ χρυσαφὶ ξέφωτο"), | |
| 101 "Greek to utf8"); | |
| 102 | |
| 103 Expect.listEquals([ | |
| 104 0xe3, 0x82, 0xa4, 0xe3, 0x83, 0xad, 0xe3, 0x83, | |
| 105 0x8f, 0xe3, 0x83, 0x8b, 0xe3, 0x83, 0x9b, 0xe3, | |
| 106 0x83, 0x98, 0xe3, 0x83, 0x88, 0x20, 0xe3, 0x83, | |
| 107 0x81, 0xe3, 0x83, 0xaa, 0xe3, 0x83, 0x8c, 0xe3, | |
| 108 0x83, 0xab, 0xe3, 0x83, 0xb2, 0x20, 0xe3, 0x83, | |
| 109 0xaf, 0xe3, 0x82, 0xab, 0xe3, 0x83, 0xa8, 0xe3, | |
| 110 0x82, 0xbf, 0xe3, 0x83, 0xac, 0xe3, 0x82, 0xbd, | |
| 111 0x20, 0xe3, 0x83, 0x84, 0xe3, 0x83, 0x8d, 0xe3, | |
| 112 0x83, 0x8a, 0xe3, 0x83, 0xa9, 0xe3, 0x83, 0xa0, | |
| 113 0x0a, 0xe3, 0x82, 0xa6, 0xe3, 0x83, 0xb0, 0xe3, | |
| 114 0x83, 0x8e, 0xe3, 0x82, 0xaa, 0xe3, 0x82, 0xaf, | |
| 115 0xe3, 0x83, 0xa4, 0xe3, 0x83, 0x9e, 0x20, 0xe3, | |
| 116 0x82, 0xb1, 0xe3, 0x83, 0x95, 0xe3, 0x82, 0xb3, | |
| 117 0xe3, 0x82, 0xa8, 0xe3, 0x83, 0x86, 0x20, 0xe3, | |
| 118 0x82, 0xa2, 0xe3, 0x82, 0xb5, 0xe3, 0x82, 0xad, | |
| 119 0xe3, 0x83, 0xa6, 0xe3, 0x83, 0xa1, 0xe3, 0x83, | |
| 120 0x9f, 0xe3, 0x82, 0xb7, 0x20, 0xe3, 0x83, 0xb1, | |
| 121 0xe3, 0x83, 0x92, 0xe3, 0x83, 0xa2, 0xe3, 0x82, | |
| 122 0xbb, 0xe3, 0x82, 0xb9, 0xe3, 0x83, 0xb3], | |
| 123 encodeAsUtf8( """ | |
| 124 イロハニホヘト チリヌルヲ ワカヨタレソ ツネナラム | |
| 125 ウヰノオクヤマ ケフコエテ アサキユメミシ ヱヒモセスン"""), | |
| 126 "Katakana"); | |
| 127 } | |
| 128 | |
| 129 void testUtf8bytesToCodepoints() { | |
| 130 Expect.listEquals([954, 972, 963, 956, 949], | |
| 131 _utf8ToCodepoints([0xce, 0xba, 0xcf, 0x8c, 0xcf, | |
| 132 0x83, 0xce, 0xbc, 0xce, 0xb5]), "κόσμε"); | |
| 133 | |
| 134 // boundary conditions: First possible sequence of a certain length | |
| 135 Expect.listEquals([0x0], _utf8ToCodepoints([0x0]), "0"); | |
| 136 Expect.listEquals([0x80], _utf8ToCodepoints([0xc2, 0x80]), "80"); | |
| 137 Expect.listEquals([0x800], | |
| 138 _utf8ToCodepoints([0xe0, 0xa0, 0x80]), "800"); | |
| 139 Expect.listEquals([0x10000], | |
| 140 _utf8ToCodepoints([0xf0, 0x90, 0x80, 0x80]), "10000"); | |
| 141 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | |
| 142 _utf8ToCodepoints([0xf8, 0x88, 0x80, 0x80, 0x80]), "200000"); | |
| 143 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | |
| 144 _utf8ToCodepoints([0xfc, 0x84, 0x80, 0x80, 0x80, 0x80]), | |
| 145 "4000000"); | |
| 146 | |
| 147 // boundary conditions: Last possible sequence of a certain length | |
| 148 Expect.listEquals([0x7f], _utf8ToCodepoints([0x7f]), "7f"); | |
| 149 Expect.listEquals([0x7ff], _utf8ToCodepoints([0xdf, 0xbf]), "7ff"); | |
| 150 Expect.listEquals([0xffff], | |
| 151 _utf8ToCodepoints([0xef, 0xbf, 0xbf]), "ffff"); | |
| 152 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | |
| 153 _utf8ToCodepoints([0xf7, 0xbf, 0xbf, 0xbf]), "1fffff"); | |
| 154 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | |
| 155 _utf8ToCodepoints([0xfb, 0xbf, 0xbf, 0xbf, 0xbf]), "3ffffff"); | |
| 156 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | |
| 157 _utf8ToCodepoints([0xfd, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf]), | |
| 158 "4000000"); | |
| 159 | |
| 160 // other boundary conditions | |
| 161 Expect.listEquals([0xd7ff], | |
| 162 _utf8ToCodepoints([0xed, 0x9f, 0xbf]), "d7ff"); | |
| 163 Expect.listEquals([0xe000], | |
| 164 _utf8ToCodepoints([0xee, 0x80, 0x80]), "e000"); | |
| 165 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | |
| 166 _utf8ToCodepoints([0xef, 0xbf, 0xbd]), "fffd"); | |
| 167 Expect.listEquals([0x10ffff], | |
| 168 _utf8ToCodepoints([0xf4, 0x8f, 0xbf, 0xbf]), "10ffff"); | |
| 169 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | |
| 170 _utf8ToCodepoints([0xf4, 0x90, 0x80, 0x80]), "110000"); | |
| 171 | |
| 172 // unexpected continuation bytes | |
| 173 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | |
| 174 _utf8ToCodepoints([0x80]), "80 => replacement character"); | |
| 175 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | |
| 176 _utf8ToCodepoints([0xbf]), "bf => replacement character"); | |
| 177 | |
| 178 List<int> allContinuationBytes = <int>[]; | |
| 179 List<int> matchingReplacementChars = <int>[]; | |
| 180 for (int i = 0x80; i < 0xc0; i++) { | |
| 181 allContinuationBytes.add(i); | |
| 182 matchingReplacementChars.add(UNICODE_REPLACEMENT_CHARACTER_CODEPOINT); | |
| 183 } | |
| 184 Expect.listEquals(matchingReplacementChars, | |
| 185 _utf8ToCodepoints(allContinuationBytes), | |
| 186 "80 - bf => replacement character x 64"); | |
| 187 | |
| 188 List<int> allFirstTwoByteSeq = <int>[]; | |
| 189 matchingReplacementChars = <int>[]; | |
| 190 for (int i = 0xc0; i < 0xe0; i++) { | |
| 191 allFirstTwoByteSeq.addAll([i, 0x20]); | |
| 192 matchingReplacementChars.addAll([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]) ; | |
|
Søren Gjesse
2012/02/01 11:25:25
Long line here and some more places below.
dcarlson
2012/02/01 22:18:46
Done.
| |
| 193 } | |
| 194 Expect.listEquals(matchingReplacementChars, | |
| 195 _utf8ToCodepoints(allFirstTwoByteSeq), | |
| 196 "c0 - df + space => replacement character + space x 32"); | |
| 197 | |
| 198 List<int> allFirstThreeByteSeq = <int>[]; | |
| 199 matchingReplacementChars = <int>[]; | |
| 200 for (int i = 0xe0; i < 0xf0; i++) { | |
| 201 allFirstThreeByteSeq.addAll([i, 0x20]); | |
| 202 matchingReplacementChars.addAll([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]) ; | |
| 203 } | |
| 204 Expect.listEquals(matchingReplacementChars, | |
| 205 _utf8ToCodepoints(allFirstThreeByteSeq), | |
| 206 "e0 - ef + space => replacement character x 16"); | |
| 207 | |
| 208 List<int> allFirstFourByteSeq = <int>[]; | |
| 209 matchingReplacementChars = <int>[]; | |
| 210 for (int i = 0xf0; i < 0xf8; i++) { | |
| 211 allFirstFourByteSeq.addAll([i, 0x20]); | |
| 212 matchingReplacementChars.addAll([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]) ; | |
| 213 } | |
| 214 Expect.listEquals(matchingReplacementChars, | |
| 215 _utf8ToCodepoints(allFirstFourByteSeq), | |
| 216 "f0 - f7 + space => replacement character x 8"); | |
| 217 | |
| 218 List<int> allFirstFiveByteSeq = <int>[]; | |
| 219 matchingReplacementChars = <int>[]; | |
| 220 for (int i = 0xf8; i < 0xfc; i++) { | |
| 221 allFirstFiveByteSeq.addAll([i, 0x20]); | |
| 222 matchingReplacementChars.addAll([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]) ; | |
| 223 } | |
| 224 Expect.listEquals(matchingReplacementChars, | |
| 225 _utf8ToCodepoints(allFirstFiveByteSeq), | |
| 226 "f8 - fb + space => replacement character x 4"); | |
| 227 | |
| 228 List<int> allFirstSixByteSeq = <int>[]; | |
| 229 matchingReplacementChars = <int>[]; | |
| 230 for (int i = 0xfc; i < 0xfe; i++) { | |
| 231 allFirstSixByteSeq.addAll([i, 0x20]); | |
| 232 matchingReplacementChars.addAll([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]) ; | |
| 233 } | |
| 234 Expect.listEquals(matchingReplacementChars, | |
| 235 _utf8ToCodepoints(allFirstSixByteSeq), | |
| 236 "fc - fd + space => replacement character x 2"); | |
| 237 | |
| 238 // Sequences with last continuation byte missing | |
| 239 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | |
| 240 _utf8ToCodepoints([0xc2]), | |
| 241 "2-byte sequence with last byte missing"); | |
| 242 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | |
| 243 _utf8ToCodepoints([0xe0, 0x80]), | |
| 244 "3-byte sequence with last byte missing"); | |
| 245 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | |
| 246 _utf8ToCodepoints([0xf0, 0x80, 0x80]), | |
| 247 "4-byte sequence with last byte missing"); | |
| 248 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | |
| 249 _utf8ToCodepoints([0xf8, 0x88, 0x80, 0x80]), | |
| 250 "5-byte sequence with last byte missing"); | |
| 251 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | |
| 252 _utf8ToCodepoints([0xfc, 0x80, 0x80, 0x80, 0x80]), | |
| 253 "6-byte sequence with last byte missing"); | |
| 254 | |
| 255 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | |
| 256 _utf8ToCodepoints([0xdf]), | |
| 257 "2-byte sequence with last byte missing (hi)"); | |
| 258 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | |
| 259 _utf8ToCodepoints([0xef, 0xbf]), | |
| 260 "3-byte sequence with last byte missing (hi)"); | |
| 261 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | |
| 262 _utf8ToCodepoints([0xf7, 0xbf, 0xbf]), | |
| 263 "4-byte sequence with last byte missing (hi)"); | |
| 264 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | |
| 265 _utf8ToCodepoints([0xfb, 0xbf, 0xbf, 0xbf]), | |
| 266 "5-byte sequence with last byte missing (hi)"); | |
| 267 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | |
| 268 _utf8ToCodepoints([0xfd, 0xbf, 0xbf, 0xbf, 0xbf]), | |
| 269 "6-byte sequence with last byte missing (hi)"); | |
| 270 | |
| 271 // Concatenation of incomplete sequences | |
| 272 Expect.listEquals( | |
| 273 [ UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, | |
| 274 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, | |
| 275 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, | |
| 276 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, | |
| 277 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, | |
| 278 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, | |
| 279 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, | |
| 280 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, | |
| 281 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, | |
| 282 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT ], | |
| 283 _utf8ToCodepoints( | |
| 284 [ 0xc2, | |
| 285 0xe0, 0x80, | |
| 286 0xf0, 0x80, 0x80, | |
| 287 0xf8, 0x88, 0x80, 0x80, | |
| 288 0xfc, 0x80, 0x80, 0x80, 0x80, | |
| 289 0xdf, | |
| 290 0xef, 0xbf, | |
| 291 0xf7, 0xbf, 0xbf, | |
| 292 0xfb, 0xbf, 0xbf, 0xbf, | |
| 293 0xfd, 0xbf, 0xbf, 0xbf, 0xbf ]), | |
| 294 "Concatenation of incomplete sequences"); | |
| 295 | |
| 296 // Impossible bytes | |
| 297 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | |
| 298 _utf8ToCodepoints([0xfe]), "fe"); | |
| 299 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | |
| 300 _utf8ToCodepoints([0xff]), "ff"); | |
| 301 Expect.listEquals([ | |
| 302 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, | |
| 303 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, | |
| 304 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, | |
| 305 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | |
| 306 _utf8ToCodepoints([0xfe, 0xfe, 0xff, 0xff]), "fe fe ff ff"); | |
| 307 | |
| 308 // Overlong sequences | |
| 309 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | |
| 310 _utf8ToCodepoints([0xc0, 0xaf]), "c0 af"); | |
| 311 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | |
| 312 _utf8ToCodepoints([0xe0, 0x80, 0xaf]), "e0 80 af"); | |
| 313 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | |
| 314 _utf8ToCodepoints([0xf0, 0x80, 0x80, 0xaf]), "f0 80 80 af"); | |
| 315 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | |
| 316 _utf8ToCodepoints([0xf8, 0x80, 0x80, 0x80, 0xaf]), "f8 80 80 80 af"); | |
| 317 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | |
| 318 _utf8ToCodepoints([0xfc, 0x80, 0x80, 0x80, 0x80, 0xaf]), | |
| 319 "fc 80 80 80 80 af"); | |
| 320 | |
| 321 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | |
| 322 _utf8ToCodepoints([0xc1, 0xbf]), "c1 bf"); | |
| 323 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | |
| 324 _utf8ToCodepoints([0xe0, 0x9f, 0xbf]), "e0 9f bf"); | |
| 325 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | |
| 326 _utf8ToCodepoints([0xf0, 0x8f, 0xbf, 0xbf]), "f0 8f bf bf"); | |
| 327 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | |
| 328 _utf8ToCodepoints([0xf8, 0x87, 0xbf, 0xbf, 0xbf]), "f8 87 bf bf bf"); | |
| 329 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | |
| 330 _utf8ToCodepoints([0xfc, 0x83, 0xbf, 0xbf, 0xbf, 0xbf]), | |
| 331 "fc 83 bf bf bf bf"); | |
| 332 | |
| 333 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | |
| 334 _utf8ToCodepoints([0xc0, 0x80]), "c0 80"); | |
| 335 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | |
| 336 _utf8ToCodepoints([0xe0, 0x80, 0x80]), "e0 80 80"); | |
| 337 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | |
| 338 _utf8ToCodepoints([0xf0, 0x80, 0x80, 0x80]), "f0 80 80 80"); | |
| 339 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | |
| 340 _utf8ToCodepoints([0xf8, 0x80, 0x80, 0x80, 0x80]), "f8 80 80 80 80"); | |
| 341 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | |
| 342 _utf8ToCodepoints([0xfc, 0x80, 0x80, 0x80, 0x80, 0x80]), | |
| 343 "fc 80 80 80 80 80"); | |
| 344 | |
| 345 // Illegal code positions | |
| 346 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | |
| 347 _utf8ToCodepoints([0xed, 0xa0, 0x80]), "U+D800"); | |
| 348 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | |
| 349 _utf8ToCodepoints([0xed, 0xad, 0xbf]), "U+DB7F"); | |
| 350 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | |
| 351 _utf8ToCodepoints([0xed, 0xae, 0x80]), "U+DB80"); | |
| 352 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | |
| 353 _utf8ToCodepoints([0xed, 0xaf, 0xbf]), "U+DBFF"); | |
| 354 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | |
| 355 _utf8ToCodepoints([0xed, 0xb0, 0x80]), "U+DC00"); | |
| 356 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | |
| 357 _utf8ToCodepoints([0xed, 0xbe, 0x80]), "U+DF80"); | |
| 358 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | |
| 359 _utf8ToCodepoints([0xed, 0xbf, 0xbf]), "U+DFFF"); | |
| 360 | |
| 361 // Paired UTF-16 surrogates | |
| 362 Expect.listEquals([ | |
| 363 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, | |
| 364 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | |
| 365 _utf8ToCodepoints([0xed, 0xa0, 0x80, 0xed, 0xb0, 0x80]), | |
| 366 "U+D800 U+DC00"); | |
| 367 Expect.listEquals([ | |
| 368 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, | |
| 369 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | |
| 370 _utf8ToCodepoints([0xed, 0xa0, 0x80, 0xed, 0xbf, 0xbf]), | |
| 371 "U+D800 U+DFFF"); | |
| 372 Expect.listEquals([ | |
| 373 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, | |
| 374 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | |
| 375 _utf8ToCodepoints([0xed, 0xad, 0xbf, 0xed, 0xb0, 0x80]), | |
| 376 "U+DB7F U+DC00"); | |
| 377 Expect.listEquals([ | |
| 378 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, | |
| 379 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | |
| 380 _utf8ToCodepoints([0xed, 0xad, 0xbf, 0xed, 0xbf, 0xbf]), | |
| 381 "U+DB7F U+DFFF"); | |
| 382 Expect.listEquals([ | |
| 383 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, | |
| 384 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | |
| 385 _utf8ToCodepoints([0xed, 0xae, 0x80, 0xed, 0xb0, 0x80]), | |
| 386 "U+DB80 U+DC00"); | |
| 387 Expect.listEquals([ | |
| 388 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, | |
| 389 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | |
| 390 _utf8ToCodepoints([0xed, 0xae, 0x80, 0xed, 0xbf, 0xbf]), | |
| 391 "U+DB80 U+DFFF"); | |
| 392 Expect.listEquals([ | |
| 393 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, | |
| 394 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | |
| 395 _utf8ToCodepoints([0xed, 0xaf, 0xbf, 0xed, 0xb0, 0x80]), | |
| 396 "U+DBFF U+DC00"); | |
| 397 Expect.listEquals([ | |
| 398 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, | |
| 399 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | |
| 400 _utf8ToCodepoints([0xed, 0xaf, 0xbf, 0xed, 0xbf, 0xbf]), | |
| 401 "U+DBFF U+DFFF"); | |
| 402 | |
| 403 // Other illegal code positions (???) | |
| 404 Expect.listEquals([0xfffe], _utf8ToCodepoints([0xef, 0xbf, 0xbe]), | |
| 405 "U+FFFE"); | |
| 406 Expect.listEquals([0xffff], _utf8ToCodepoints([0xef, 0xbf, 0xbf]), | |
| 407 "U+FFFF"); | |
| 408 } | |
| 409 | |
| 410 void testUtf8BytesToString() { | |
| 411 Expect.stringEquals("The quick brown fox jumps over the lazy dog.", | |
| 412 decodeFromUtf8([ | |
| 413 0x54, 0x68, 0x65, 0x20, 0x71, 0x75, 0x69, 0x63, | |
| 414 0x6b, 0x20, 0x62, 0x72, 0x6f, 0x77, 0x6e, 0x20, | |
| 415 0x66, 0x6f, 0x78, 0x20, 0x6a, 0x75, 0x6d, 0x70, | |
| 416 0x73, 0x20, 0x6f, 0x76, 0x65, 0x72, 0x20, 0x74, | |
| 417 0x68, 0x65, 0x20, 0x6c, 0x61, 0x7a, 0x79, 0x20, | |
| 418 0x64, 0x6f, 0x67, 0x2e]), "English"); | |
| 419 | |
| 420 Expect.stringEquals("Quizdeltagerne spiste jordbær med fløde mens " + | |
| 421 "cirkusklovnen Wolther spillede på xylofon.", | |
| 422 decodeFromUtf8([ | |
| 423 0x51, 0x75, 0x69, 0x7a, 0x64, 0x65, 0x6c, 0x74, | |
| 424 0x61, 0x67, 0x65, 0x72, 0x6e, 0x65, 0x20, 0x73, | |
| 425 0x70, 0x69, 0x73, 0x74, 0x65, 0x20, 0x6a, 0x6f, | |
| 426 0x72, 0x64, 0x62, 0xc3, 0xa6, 0x72, 0x20, 0x6d, | |
| 427 0x65, 0x64, 0x20, 0x66, 0x6c, 0xc3, 0xb8, 0x64, | |
| 428 0x65, 0x20, 0x6d, 0x65, 0x6e, 0x73, 0x20, 0x63, | |
| 429 0x69, 0x72, 0x6b, 0x75, 0x73, 0x6b, 0x6c, 0x6f, | |
| 430 0x76, 0x6e, 0x65, 0x6e, 0x20, 0x57, 0x6f, 0x6c, | |
| 431 0x74, 0x68, 0x65, 0x72, 0x20, 0x73, 0x70, 0x69, | |
| 432 0x6c, 0x6c, 0x65, 0x64, 0x65, 0x20, 0x70, 0xc3, | |
| 433 0xa5, 0x20, 0x78, 0x79, 0x6c, 0x6f, 0x66, 0x6f, | |
| 434 0x6e, 0x2e]), "Danish"); | |
| 435 | |
| 436 Expect.stringEquals("דג סקרן שט בים מאוכזב ולפתע מצא לו חברה איך הקליטה", | |
| 437 decodeFromUtf8([ | |
| 438 0xd7, 0x93, 0xd7, 0x92, 0x20, 0xd7, 0xa1, 0xd7, | |
| 439 0xa7, 0xd7, 0xa8, 0xd7, 0x9f, 0x20, 0xd7, 0xa9, | |
| 440 0xd7, 0x98, 0x20, 0xd7, 0x91, 0xd7, 0x99, 0xd7, | |
| 441 0x9d, 0x20, 0xd7, 0x9e, 0xd7, 0x90, 0xd7, 0x95, | |
| 442 0xd7, 0x9b, 0xd7, 0x96, 0xd7, 0x91, 0x20, 0xd7, | |
| 443 0x95, 0xd7, 0x9c, 0xd7, 0xa4, 0xd7, 0xaa, 0xd7, | |
| 444 0xa2, 0x20, 0xd7, 0x9e, 0xd7, 0xa6, 0xd7, 0x90, | |
| 445 0x20, 0xd7, 0x9c, 0xd7, 0x95, 0x20, 0xd7, 0x97, | |
| 446 0xd7, 0x91, 0xd7, 0xa8, 0xd7, 0x94, 0x20, 0xd7, | |
| 447 0x90, 0xd7, 0x99, 0xd7, 0x9a, 0x20, 0xd7, 0x94, | |
| 448 0xd7, 0xa7, 0xd7, 0x9c, 0xd7, 0x99, 0xd7, 0x98, | |
| 449 0xd7, 0x94]), "Hebrew"); | |
| 450 | |
| 451 Expect.stringEquals("Съешь же ещё этих мягких французских булок да выпей " + | |
| 452 "чаю", | |
| 453 decodeFromUtf8([ | |
| 454 0xd0, 0xa1, 0xd1, 0x8a, 0xd0 ,0xb5, 0xd1, 0x88, | |
| 455 0xd1, 0x8c, 0x20, 0xd0, 0xb6, 0xd0, 0xb5, 0x20, | |
| 456 0xd0, 0xb5, 0xd1, 0x89, 0xd1, 0x91, 0x20, 0xd1, | |
| 457 0x8d, 0xd1, 0x82, 0xd0, 0xb8, 0xd1, 0x85, 0x20, | |
| 458 0xd0, 0xbc, 0xd1, 0x8f, 0xd0, 0xb3, 0xd0, 0xba, | |
| 459 0xd0, 0xb8, 0xd1, 0x85, 0x20, 0xd1, 0x84, 0xd1, | |
| 460 0x80, 0xd0, 0xb0, 0xd0, 0xbd, 0xd1, 0x86, 0xd1, | |
| 461 0x83, 0xd0, 0xb7, 0xd1, 0x81, 0xd0, 0xba, 0xd0, | |
| 462 0xb8, 0xd1, 0x85, 0x20, 0xd0, 0xb1, 0xd1, 0x83, | |
| 463 0xd0, 0xbb, 0xd0, 0xbe, 0xd0, 0xba, 0x20, 0xd0, | |
| 464 0xb4, 0xd0, 0xb0, 0x20, 0xd0, 0xb2, 0xd1, 0x8b, | |
| 465 0xd0, 0xbf, 0xd0, 0xb5, 0xd0, 0xb9, 0x20, 0xd1, | |
| 466 0x87, 0xd0, 0xb0, 0xd1, 0x8e]), "Russian #2"); | |
| 467 | |
| 468 Expect.stringEquals("Γαζέες καὶ μυρτιὲς δὲν θὰ βρῶ πιὰ στὸ χρυσαφὶ ξέφωτο", | |
| 469 decodeFromUtf8([ | |
| 470 0xce, 0x93, 0xce, 0xb1, 0xce, 0xb6, 0xce, 0xad, | |
| 471 0xce, 0xb5, 0xcf, 0x82, 0x20, 0xce, 0xba, 0xce, | |
| 472 0xb1, 0xe1, 0xbd, 0xb6, 0x20, 0xce, 0xbc, 0xcf, | |
| 473 0x85, 0xcf, 0x81, 0xcf, 0x84, 0xce, 0xb9, 0xe1, | |
| 474 0xbd, 0xb2, 0xcf, 0x82, 0x20, 0xce, 0xb4, 0xe1, | |
| 475 0xbd, 0xb2, 0xce, 0xbd, 0x20, 0xce, 0xb8, 0xe1, | |
| 476 0xbd, 0xb0, 0x20, 0xce, 0xb2, 0xcf, 0x81, 0xe1, | |
| 477 0xbf, 0xb6, 0x20, 0xcf, 0x80, 0xce, 0xb9, 0xe1, | |
| 478 0xbd, 0xb0, 0x20, 0xcf, 0x83, 0xcf, 0x84, 0xe1, | |
| 479 0xbd, 0xb8, 0x20, 0xcf, 0x87, 0xcf, 0x81, 0xcf, | |
| 480 0x85, 0xcf, 0x83, 0xce, 0xb1, 0xcf, 0x86, 0xe1, | |
| 481 0xbd, 0xb6, 0x20, 0xce, 0xbe, 0xce, 0xad, 0xcf, | |
| 482 0x86, 0xcf, 0x89, 0xcf, 0x84, 0xce, 0xbf]), "Greek"); | |
| 483 Expect.stringEquals(""" | |
| 484 イロハニホヘト チリヌルヲ ワカヨタレソ ツネナラム | |
| 485 ウヰノオクヤマ ケフコエテ アサキユメミシ ヱヒモセスン""", | |
| 486 decodeFromUtf8([ | |
| 487 0xe3, 0x82, 0xa4, 0xe3, 0x83, 0xad, 0xe3, 0x83, | |
| 488 0x8f, 0xe3, 0x83, 0x8b, 0xe3, 0x83, 0x9b, 0xe3, | |
| 489 0x83, 0x98, 0xe3, 0x83, 0x88, 0x20, 0xe3, 0x83, | |
| 490 0x81, 0xe3, 0x83, 0xaa, 0xe3, 0x83, 0x8c, 0xe3, | |
| 491 0x83, 0xab, 0xe3, 0x83, 0xb2, 0x20, 0xe3, 0x83, | |
| 492 0xaf, 0xe3, 0x82, 0xab, 0xe3, 0x83, 0xa8, 0xe3, | |
| 493 0x82, 0xbf, 0xe3, 0x83, 0xac, 0xe3, 0x82, 0xbd, | |
| 494 0x20, 0xe3, 0x83, 0x84, 0xe3, 0x83, 0x8d, 0xe3, | |
| 495 0x83, 0x8a, 0xe3, 0x83, 0xa9, 0xe3, 0x83, 0xa0, | |
| 496 0x0a, 0xe3, 0x82, 0xa6, 0xe3, 0x83, 0xb0, 0xe3, | |
| 497 0x83, 0x8e, 0xe3, 0x82, 0xaa, 0xe3, 0x82, 0xaf, | |
| 498 0xe3, 0x83, 0xa4, 0xe3, 0x83, 0x9e, 0x20, 0xe3, | |
| 499 0x82, 0xb1, 0xe3, 0x83, 0x95, 0xe3, 0x82, 0xb3, | |
| 500 0xe3, 0x82, 0xa8, 0xe3, 0x83, 0x86, 0x20, 0xe3, | |
| 501 0x82, 0xa2, 0xe3, 0x82, 0xb5, 0xe3, 0x82, 0xad, | |
| 502 0xe3, 0x83, 0xa6, 0xe3, 0x83, 0xa1, 0xe3, 0x83, | |
| 503 0x9f, 0xe3, 0x82, 0xb7, 0x20, 0xe3, 0x83, 0xb1, | |
| 504 0xe3, 0x83, 0x92, 0xe3, 0x83, 0xa2, 0xe3, 0x82, | |
| 505 0xbb, 0xe3, 0x82, 0xb9, 0xe3, 0x83, 0xb3]), "Katakana"); | |
| 506 } | |
| 507 } | |
| OLD | NEW |
