Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(194)

Side by Side Diff: utils/tests/string_encoding/utf8_tests.dart

Issue 9462001: Unify most of our utf8 implementations. This takes the implementation (Closed) Base URL: http://dart.googlecode.com/svn/branches/bleeding_edge/dart/
Patch Set: Created 8 years, 9 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch | Annotate | Revision Log
« no previous file with comments | « utils/tests/string_encoding/utf8_benchmarks.dart ('k') | no next file » | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 #!/usr/bin/env dart 1 #!/usr/bin/env dart
2 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file 2 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file
3 // for details. All rights reserved. Use of this source code is governed by a 3 // for details. All rights reserved. Use of this source code is governed by a
4 // BSD-style license that can be found in the LICENSE file. 4 // BSD-style license that can be found in the LICENSE file.
5 5
6 #library("utf8_tests"); 6 #library("utf8_tests");
7 #import("dunit.dart"); 7 #import("dunit.dart");
8 #import("../../string_encoding/unicode.dart"); 8 #import("../../../lib/utf/utf.dart");
9 #import("../../string_encoding/unicode_core.dart");
10 #source("../../string_encoding/utf8_impl.dart");
11 9
12 void main() { 10 void main() {
13 TestSuite suite = new TestSuite(); 11 TestSuite suite = new TestSuite();
14 suite.registerTestClass(new Utf8Tests()); 12 suite.registerTestClass(new Utf8Tests());
15 suite.run(); 13 suite.run();
16 } 14 }
17 15
18 class Utf8Tests extends TestClass { 16 class Utf8Tests extends TestClass {
19 static final String testEnglishPhrase = 17 static final String testEnglishPhrase =
20 "The quick brown fox jumps over the lazy dog."; 18 "The quick brown fox jumps over the lazy dog.";
(...skipping 125 matching lines...) Expand 10 before | Expand all | Expand 10 after
146 144
147 Expect.listEquals(testGreekUtf8, encodeUtf8(testGreekPhrase), 145 Expect.listEquals(testGreekUtf8, encodeUtf8(testGreekPhrase),
148 "Greek to utf8"); 146 "Greek to utf8");
149 147
150 Expect.listEquals(testKatakanaUtf8, encodeUtf8(testKatakanaPhrase), 148 Expect.listEquals(testKatakanaUtf8, encodeUtf8(testKatakanaPhrase),
151 "Katakana to utf8"); 149 "Katakana to utf8");
152 } 150 }
153 151
154 void testUtf8bytesToCodepoints() { 152 void testUtf8bytesToCodepoints() {
155 Expect.listEquals([954, 972, 963, 956, 949], 153 Expect.listEquals([954, 972, 963, 956, 949],
156 _utf8ToCodepoints([0xce, 0xba, 0xcf, 0x8c, 0xcf, 154 utf8ToCodepoints([0xce, 0xba, 0xcf, 0x8c, 0xcf,
157 0x83, 0xce, 0xbc, 0xce, 0xb5]), "κόσμε"); 155 0x83, 0xce, 0xbc, 0xce, 0xb5]), "κόσμε");
158 156
159 // boundary conditions: First possible sequence of a certain length 157 // boundary conditions: First possible sequence of a certain length
160 Expect.listEquals([], _utf8ToCodepoints([]), "no input"); 158 Expect.listEquals([], utf8ToCodepoints([]), "no input");
161 Expect.listEquals([0x0], _utf8ToCodepoints([0x0]), "0"); 159 Expect.listEquals([0x0], utf8ToCodepoints([0x0]), "0");
162 Expect.listEquals([0x80], _utf8ToCodepoints([0xc2, 0x80]), "80"); 160 Expect.listEquals([0x80], utf8ToCodepoints([0xc2, 0x80]), "80");
163 Expect.listEquals([0x800], 161 Expect.listEquals([0x800],
164 _utf8ToCodepoints([0xe0, 0xa0, 0x80]), "800"); 162 utf8ToCodepoints([0xe0, 0xa0, 0x80]), "800");
165 Expect.listEquals([0x10000], 163 Expect.listEquals([0x10000],
166 _utf8ToCodepoints([0xf0, 0x90, 0x80, 0x80]), "10000"); 164 utf8ToCodepoints([0xf0, 0x90, 0x80, 0x80]), "10000");
167 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], 165 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
168 _utf8ToCodepoints([0xf8, 0x88, 0x80, 0x80, 0x80]), "200000"); 166 utf8ToCodepoints([0xf8, 0x88, 0x80, 0x80, 0x80]), "200000");
169 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], 167 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
170 _utf8ToCodepoints([0xfc, 0x84, 0x80, 0x80, 0x80, 0x80]), 168 utf8ToCodepoints([0xfc, 0x84, 0x80, 0x80, 0x80, 0x80]),
171 "4000000"); 169 "4000000");
172 170
173 // boundary conditions: Last possible sequence of a certain length 171 // boundary conditions: Last possible sequence of a certain length
174 Expect.listEquals([0x7f], _utf8ToCodepoints([0x7f]), "7f"); 172 Expect.listEquals([0x7f], utf8ToCodepoints([0x7f]), "7f");
175 Expect.listEquals([0x7ff], _utf8ToCodepoints([0xdf, 0xbf]), "7ff"); 173 Expect.listEquals([0x7ff], utf8ToCodepoints([0xdf, 0xbf]), "7ff");
176 Expect.listEquals([0xffff], 174 Expect.listEquals([0xffff],
177 _utf8ToCodepoints([0xef, 0xbf, 0xbf]), "ffff"); 175 utf8ToCodepoints([0xef, 0xbf, 0xbf]), "ffff");
178 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], 176 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
179 _utf8ToCodepoints([0xf7, 0xbf, 0xbf, 0xbf]), "1fffff"); 177 utf8ToCodepoints([0xf7, 0xbf, 0xbf, 0xbf]), "1fffff");
180 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], 178 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
181 _utf8ToCodepoints([0xfb, 0xbf, 0xbf, 0xbf, 0xbf]), "3ffffff"); 179 utf8ToCodepoints([0xfb, 0xbf, 0xbf, 0xbf, 0xbf]), "3ffffff");
182 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], 180 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
183 _utf8ToCodepoints([0xfd, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf]), 181 utf8ToCodepoints([0xfd, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf]),
184 "4000000"); 182 "4000000");
185 183
186 // other boundary conditions 184 // other boundary conditions
187 Expect.listEquals([0xd7ff], 185 Expect.listEquals([0xd7ff],
188 _utf8ToCodepoints([0xed, 0x9f, 0xbf]), "d7ff"); 186 utf8ToCodepoints([0xed, 0x9f, 0xbf]), "d7ff");
189 Expect.listEquals([0xe000], 187 Expect.listEquals([0xe000],
190 _utf8ToCodepoints([0xee, 0x80, 0x80]), "e000"); 188 utf8ToCodepoints([0xee, 0x80, 0x80]), "e000");
191 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], 189 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
192 _utf8ToCodepoints([0xef, 0xbf, 0xbd]), "fffd"); 190 utf8ToCodepoints([0xef, 0xbf, 0xbd]), "fffd");
193 Expect.listEquals([0x10ffff], 191 Expect.listEquals([0x10ffff],
194 _utf8ToCodepoints([0xf4, 0x8f, 0xbf, 0xbf]), "10ffff"); 192 utf8ToCodepoints([0xf4, 0x8f, 0xbf, 0xbf]), "10ffff");
195 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], 193 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
196 _utf8ToCodepoints([0xf4, 0x90, 0x80, 0x80]), "110000"); 194 utf8ToCodepoints([0xf4, 0x90, 0x80, 0x80]), "110000");
197 195
198 // unexpected continuation bytes 196 // unexpected continuation bytes
199 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], 197 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
200 _utf8ToCodepoints([0x80]), "80 => replacement character"); 198 utf8ToCodepoints([0x80]), "80 => replacement character");
201 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], 199 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
202 _utf8ToCodepoints([0xbf]), "bf => replacement character"); 200 utf8ToCodepoints([0xbf]), "bf => replacement character");
203 201
204 List<int> allContinuationBytes = <int>[]; 202 List<int> allContinuationBytes = <int>[];
205 List<int> matchingReplacementChars = <int>[]; 203 List<int> matchingReplacementChars = <int>[];
206 for (int i = 0x80; i < 0xc0; i++) { 204 for (int i = 0x80; i < 0xc0; i++) {
207 allContinuationBytes.add(i); 205 allContinuationBytes.add(i);
208 matchingReplacementChars.add(UNICODE_REPLACEMENT_CHARACTER_CODEPOINT); 206 matchingReplacementChars.add(UNICODE_REPLACEMENT_CHARACTER_CODEPOINT);
209 } 207 }
210 Expect.listEquals(matchingReplacementChars, 208 Expect.listEquals(matchingReplacementChars,
211 _utf8ToCodepoints(allContinuationBytes), 209 utf8ToCodepoints(allContinuationBytes),
212 "80 - bf => replacement character x 64"); 210 "80 - bf => replacement character x 64");
213 211
214 List<int> allFirstTwoByteSeq = <int>[]; 212 List<int> allFirstTwoByteSeq = <int>[];
215 matchingReplacementChars = <int>[]; 213 matchingReplacementChars = <int>[];
216 for (int i = 0xc0; i < 0xe0; i++) { 214 for (int i = 0xc0; i < 0xe0; i++) {
217 allFirstTwoByteSeq.addAll([i, 0x20]); 215 allFirstTwoByteSeq.addAll([i, 0x20]);
218 matchingReplacementChars.addAll( 216 matchingReplacementChars.addAll(
219 [UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]); 217 [UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]);
220 } 218 }
221 Expect.listEquals(matchingReplacementChars, 219 Expect.listEquals(matchingReplacementChars,
222 _utf8ToCodepoints(allFirstTwoByteSeq), 220 utf8ToCodepoints(allFirstTwoByteSeq),
223 "c0 - df + space => replacement character + space x 32"); 221 "c0 - df + space => replacement character + space x 32");
224 222
225 List<int> allFirstThreeByteSeq = <int>[]; 223 List<int> allFirstThreeByteSeq = <int>[];
226 matchingReplacementChars = <int>[]; 224 matchingReplacementChars = <int>[];
227 for (int i = 0xe0; i < 0xf0; i++) { 225 for (int i = 0xe0; i < 0xf0; i++) {
228 allFirstThreeByteSeq.addAll([i, 0x20]); 226 allFirstThreeByteSeq.addAll([i, 0x20]);
229 matchingReplacementChars.addAll( 227 matchingReplacementChars.addAll(
230 [UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]); 228 [UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]);
231 } 229 }
232 Expect.listEquals(matchingReplacementChars, 230 Expect.listEquals(matchingReplacementChars,
233 _utf8ToCodepoints(allFirstThreeByteSeq), 231 utf8ToCodepoints(allFirstThreeByteSeq),
234 "e0 - ef + space => replacement character x 16"); 232 "e0 - ef + space => replacement character x 16");
235 233
236 List<int> allFirstFourByteSeq = <int>[]; 234 List<int> allFirstFourByteSeq = <int>[];
237 matchingReplacementChars = <int>[]; 235 matchingReplacementChars = <int>[];
238 for (int i = 0xf0; i < 0xf8; i++) { 236 for (int i = 0xf0; i < 0xf8; i++) {
239 allFirstFourByteSeq.addAll([i, 0x20]); 237 allFirstFourByteSeq.addAll([i, 0x20]);
240 matchingReplacementChars.addAll( 238 matchingReplacementChars.addAll(
241 [UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]); 239 [UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]);
242 } 240 }
243 Expect.listEquals(matchingReplacementChars, 241 Expect.listEquals(matchingReplacementChars,
244 _utf8ToCodepoints(allFirstFourByteSeq), 242 utf8ToCodepoints(allFirstFourByteSeq),
245 "f0 - f7 + space => replacement character x 8"); 243 "f0 - f7 + space => replacement character x 8");
246 244
247 List<int> allFirstFiveByteSeq = <int>[]; 245 List<int> allFirstFiveByteSeq = <int>[];
248 matchingReplacementChars = <int>[]; 246 matchingReplacementChars = <int>[];
249 for (int i = 0xf8; i < 0xfc; i++) { 247 for (int i = 0xf8; i < 0xfc; i++) {
250 allFirstFiveByteSeq.addAll([i, 0x20]); 248 allFirstFiveByteSeq.addAll([i, 0x20]);
251 matchingReplacementChars.addAll( 249 matchingReplacementChars.addAll(
252 [UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]); 250 [UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]);
253 } 251 }
254 Expect.listEquals(matchingReplacementChars, 252 Expect.listEquals(matchingReplacementChars,
255 _utf8ToCodepoints(allFirstFiveByteSeq), 253 utf8ToCodepoints(allFirstFiveByteSeq),
256 "f8 - fb + space => replacement character x 4"); 254 "f8 - fb + space => replacement character x 4");
257 255
258 List<int> allFirstSixByteSeq = <int>[]; 256 List<int> allFirstSixByteSeq = <int>[];
259 matchingReplacementChars = <int>[]; 257 matchingReplacementChars = <int>[];
260 for (int i = 0xfc; i < 0xfe; i++) { 258 for (int i = 0xfc; i < 0xfe; i++) {
261 allFirstSixByteSeq.addAll([i, 0x20]); 259 allFirstSixByteSeq.addAll([i, 0x20]);
262 matchingReplacementChars.addAll( 260 matchingReplacementChars.addAll(
263 [UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]); 261 [UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]);
264 } 262 }
265 Expect.listEquals(matchingReplacementChars, 263 Expect.listEquals(matchingReplacementChars,
266 _utf8ToCodepoints(allFirstSixByteSeq), 264 utf8ToCodepoints(allFirstSixByteSeq),
267 "fc - fd + space => replacement character x 2"); 265 "fc - fd + space => replacement character x 2");
268 266
269 // Sequences with last continuation byte missing 267 // Sequences with last continuation byte missing
270 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], 268 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
271 _utf8ToCodepoints([0xc2]), 269 utf8ToCodepoints([0xc2]),
272 "2-byte sequence with last byte missing"); 270 "2-byte sequence with last byte missing");
273 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], 271 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
274 _utf8ToCodepoints([0xe0, 0x80]), 272 utf8ToCodepoints([0xe0, 0x80]),
275 "3-byte sequence with last byte missing"); 273 "3-byte sequence with last byte missing");
276 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], 274 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
277 _utf8ToCodepoints([0xf0, 0x80, 0x80]), 275 utf8ToCodepoints([0xf0, 0x80, 0x80]),
278 "4-byte sequence with last byte missing"); 276 "4-byte sequence with last byte missing");
279 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], 277 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
280 _utf8ToCodepoints([0xf8, 0x88, 0x80, 0x80]), 278 utf8ToCodepoints([0xf8, 0x88, 0x80, 0x80]),
281 "5-byte sequence with last byte missing"); 279 "5-byte sequence with last byte missing");
282 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], 280 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
283 _utf8ToCodepoints([0xfc, 0x80, 0x80, 0x80, 0x80]), 281 utf8ToCodepoints([0xfc, 0x80, 0x80, 0x80, 0x80]),
284 "6-byte sequence with last byte missing"); 282 "6-byte sequence with last byte missing");
285 283
286 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], 284 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
287 _utf8ToCodepoints([0xdf]), 285 utf8ToCodepoints([0xdf]),
288 "2-byte sequence with last byte missing (hi)"); 286 "2-byte sequence with last byte missing (hi)");
289 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], 287 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
290 _utf8ToCodepoints([0xef, 0xbf]), 288 utf8ToCodepoints([0xef, 0xbf]),
291 "3-byte sequence with last byte missing (hi)"); 289 "3-byte sequence with last byte missing (hi)");
292 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], 290 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
293 _utf8ToCodepoints([0xf7, 0xbf, 0xbf]), 291 utf8ToCodepoints([0xf7, 0xbf, 0xbf]),
294 "4-byte sequence with last byte missing (hi)"); 292 "4-byte sequence with last byte missing (hi)");
295 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], 293 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
296 _utf8ToCodepoints([0xfb, 0xbf, 0xbf, 0xbf]), 294 utf8ToCodepoints([0xfb, 0xbf, 0xbf, 0xbf]),
297 "5-byte sequence with last byte missing (hi)"); 295 "5-byte sequence with last byte missing (hi)");
298 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], 296 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
299 _utf8ToCodepoints([0xfd, 0xbf, 0xbf, 0xbf, 0xbf]), 297 utf8ToCodepoints([0xfd, 0xbf, 0xbf, 0xbf, 0xbf]),
300 "6-byte sequence with last byte missing (hi)"); 298 "6-byte sequence with last byte missing (hi)");
301 299
302 // Concatenation of incomplete sequences 300 // Concatenation of incomplete sequences
303 Expect.listEquals( 301 Expect.listEquals(
304 [ UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, 302 [ UNICODE_REPLACEMENT_CHARACTER_CODEPOINT,
305 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, 303 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT,
306 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, 304 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT,
307 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, 305 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT,
308 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, 306 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT,
309 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, 307 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT,
310 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, 308 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT,
311 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, 309 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT,
312 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, 310 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT,
313 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT ], 311 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT ],
314 _utf8ToCodepoints( 312 utf8ToCodepoints(
315 [ 0xc2, 313 [ 0xc2,
316 0xe0, 0x80, 314 0xe0, 0x80,
317 0xf0, 0x80, 0x80, 315 0xf0, 0x80, 0x80,
318 0xf8, 0x88, 0x80, 0x80, 316 0xf8, 0x88, 0x80, 0x80,
319 0xfc, 0x80, 0x80, 0x80, 0x80, 317 0xfc, 0x80, 0x80, 0x80, 0x80,
320 0xdf, 318 0xdf,
321 0xef, 0xbf, 319 0xef, 0xbf,
322 0xf7, 0xbf, 0xbf, 320 0xf7, 0xbf, 0xbf,
323 0xfb, 0xbf, 0xbf, 0xbf, 321 0xfb, 0xbf, 0xbf, 0xbf,
324 0xfd, 0xbf, 0xbf, 0xbf, 0xbf ]), 322 0xfd, 0xbf, 0xbf, 0xbf, 0xbf ]),
325 "Concatenation of incomplete sequences"); 323 "Concatenation of incomplete sequences");
326 324
327 // Impossible bytes 325 // Impossible bytes
328 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], 326 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
329 _utf8ToCodepoints([0xfe]), "fe"); 327 utf8ToCodepoints([0xfe]), "fe");
330 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], 328 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
331 _utf8ToCodepoints([0xff]), "ff"); 329 utf8ToCodepoints([0xff]), "ff");
332 Expect.listEquals([ 330 Expect.listEquals([
333 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, 331 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT,
334 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, 332 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT,
335 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, 333 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT,
336 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], 334 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
337 _utf8ToCodepoints([0xfe, 0xfe, 0xff, 0xff]), "fe fe ff ff"); 335 utf8ToCodepoints([0xfe, 0xfe, 0xff, 0xff]), "fe fe ff ff");
338 336
339 // Overlong sequences 337 // Overlong sequences
340 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], 338 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
341 _utf8ToCodepoints([0xc0, 0xaf]), "c0 af"); 339 utf8ToCodepoints([0xc0, 0xaf]), "c0 af");
342 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], 340 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
343 _utf8ToCodepoints([0xe0, 0x80, 0xaf]), "e0 80 af"); 341 utf8ToCodepoints([0xe0, 0x80, 0xaf]), "e0 80 af");
344 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], 342 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
345 _utf8ToCodepoints([0xf0, 0x80, 0x80, 0xaf]), "f0 80 80 af"); 343 utf8ToCodepoints([0xf0, 0x80, 0x80, 0xaf]), "f0 80 80 af");
346 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], 344 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
347 _utf8ToCodepoints([0xf8, 0x80, 0x80, 0x80, 0xaf]), "f8 80 80 80 af"); 345 utf8ToCodepoints([0xf8, 0x80, 0x80, 0x80, 0xaf]), "f8 80 80 80 af");
348 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], 346 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
349 _utf8ToCodepoints([0xfc, 0x80, 0x80, 0x80, 0x80, 0xaf]), 347 utf8ToCodepoints([0xfc, 0x80, 0x80, 0x80, 0x80, 0xaf]),
350 "fc 80 80 80 80 af"); 348 "fc 80 80 80 80 af");
351 349
352 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], 350 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
353 _utf8ToCodepoints([0xc1, 0xbf]), "c1 bf"); 351 utf8ToCodepoints([0xc1, 0xbf]), "c1 bf");
354 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], 352 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
355 _utf8ToCodepoints([0xe0, 0x9f, 0xbf]), "e0 9f bf"); 353 utf8ToCodepoints([0xe0, 0x9f, 0xbf]), "e0 9f bf");
356 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], 354 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
357 _utf8ToCodepoints([0xf0, 0x8f, 0xbf, 0xbf]), "f0 8f bf bf"); 355 utf8ToCodepoints([0xf0, 0x8f, 0xbf, 0xbf]), "f0 8f bf bf");
358 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], 356 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
359 _utf8ToCodepoints([0xf8, 0x87, 0xbf, 0xbf, 0xbf]), "f8 87 bf bf bf"); 357 utf8ToCodepoints([0xf8, 0x87, 0xbf, 0xbf, 0xbf]), "f8 87 bf bf bf");
360 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], 358 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
361 _utf8ToCodepoints([0xfc, 0x83, 0xbf, 0xbf, 0xbf, 0xbf]), 359 utf8ToCodepoints([0xfc, 0x83, 0xbf, 0xbf, 0xbf, 0xbf]),
362 "fc 83 bf bf bf bf"); 360 "fc 83 bf bf bf bf");
363 361
364 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], 362 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
365 _utf8ToCodepoints([0xc0, 0x80]), "c0 80"); 363 utf8ToCodepoints([0xc0, 0x80]), "c0 80");
366 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], 364 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
367 _utf8ToCodepoints([0xe0, 0x80, 0x80]), "e0 80 80"); 365 utf8ToCodepoints([0xe0, 0x80, 0x80]), "e0 80 80");
368 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], 366 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
369 _utf8ToCodepoints([0xf0, 0x80, 0x80, 0x80]), "f0 80 80 80"); 367 utf8ToCodepoints([0xf0, 0x80, 0x80, 0x80]), "f0 80 80 80");
370 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], 368 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
371 _utf8ToCodepoints([0xf8, 0x80, 0x80, 0x80, 0x80]), "f8 80 80 80 80"); 369 utf8ToCodepoints([0xf8, 0x80, 0x80, 0x80, 0x80]), "f8 80 80 80 80");
372 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], 370 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
373 _utf8ToCodepoints([0xfc, 0x80, 0x80, 0x80, 0x80, 0x80]), 371 utf8ToCodepoints([0xfc, 0x80, 0x80, 0x80, 0x80, 0x80]),
374 "fc 80 80 80 80 80"); 372 "fc 80 80 80 80 80");
375 373
376 // Illegal code positions 374 // Illegal code positions
377 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], 375 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
378 _utf8ToCodepoints([0xed, 0xa0, 0x80]), "U+D800"); 376 utf8ToCodepoints([0xed, 0xa0, 0x80]), "U+D800");
379 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], 377 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
380 _utf8ToCodepoints([0xed, 0xad, 0xbf]), "U+DB7F"); 378 utf8ToCodepoints([0xed, 0xad, 0xbf]), "U+DB7F");
381 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], 379 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
382 _utf8ToCodepoints([0xed, 0xae, 0x80]), "U+DB80"); 380 utf8ToCodepoints([0xed, 0xae, 0x80]), "U+DB80");
383 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], 381 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
384 _utf8ToCodepoints([0xed, 0xaf, 0xbf]), "U+DBFF"); 382 utf8ToCodepoints([0xed, 0xaf, 0xbf]), "U+DBFF");
385 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], 383 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
386 _utf8ToCodepoints([0xed, 0xb0, 0x80]), "U+DC00"); 384 utf8ToCodepoints([0xed, 0xb0, 0x80]), "U+DC00");
387 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], 385 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
388 _utf8ToCodepoints([0xed, 0xbe, 0x80]), "U+DF80"); 386 utf8ToCodepoints([0xed, 0xbe, 0x80]), "U+DF80");
389 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], 387 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
390 _utf8ToCodepoints([0xed, 0xbf, 0xbf]), "U+DFFF"); 388 utf8ToCodepoints([0xed, 0xbf, 0xbf]), "U+DFFF");
391 389
392 // Paired UTF-16 surrogates 390 // Paired UTF-16 surrogates
393 Expect.listEquals([ 391 Expect.listEquals([
394 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, 392 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT,
395 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], 393 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
396 _utf8ToCodepoints([0xed, 0xa0, 0x80, 0xed, 0xb0, 0x80]), 394 utf8ToCodepoints([0xed, 0xa0, 0x80, 0xed, 0xb0, 0x80]),
397 "U+D800 U+DC00"); 395 "U+D800 U+DC00");
398 Expect.listEquals([ 396 Expect.listEquals([
399 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, 397 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT,
400 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], 398 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
401 _utf8ToCodepoints([0xed, 0xa0, 0x80, 0xed, 0xbf, 0xbf]), 399 utf8ToCodepoints([0xed, 0xa0, 0x80, 0xed, 0xbf, 0xbf]),
402 "U+D800 U+DFFF"); 400 "U+D800 U+DFFF");
403 Expect.listEquals([ 401 Expect.listEquals([
404 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, 402 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT,
405 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], 403 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
406 _utf8ToCodepoints([0xed, 0xad, 0xbf, 0xed, 0xb0, 0x80]), 404 utf8ToCodepoints([0xed, 0xad, 0xbf, 0xed, 0xb0, 0x80]),
407 "U+DB7F U+DC00"); 405 "U+DB7F U+DC00");
408 Expect.listEquals([ 406 Expect.listEquals([
409 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, 407 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT,
410 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], 408 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
411 _utf8ToCodepoints([0xed, 0xad, 0xbf, 0xed, 0xbf, 0xbf]), 409 utf8ToCodepoints([0xed, 0xad, 0xbf, 0xed, 0xbf, 0xbf]),
412 "U+DB7F U+DFFF"); 410 "U+DB7F U+DFFF");
413 Expect.listEquals([ 411 Expect.listEquals([
414 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, 412 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT,
415 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], 413 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
416 _utf8ToCodepoints([0xed, 0xae, 0x80, 0xed, 0xb0, 0x80]), 414 utf8ToCodepoints([0xed, 0xae, 0x80, 0xed, 0xb0, 0x80]),
417 "U+DB80 U+DC00"); 415 "U+DB80 U+DC00");
418 Expect.listEquals([ 416 Expect.listEquals([
419 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, 417 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT,
420 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], 418 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
421 _utf8ToCodepoints([0xed, 0xae, 0x80, 0xed, 0xbf, 0xbf]), 419 utf8ToCodepoints([0xed, 0xae, 0x80, 0xed, 0xbf, 0xbf]),
422 "U+DB80 U+DFFF"); 420 "U+DB80 U+DFFF");
423 Expect.listEquals([ 421 Expect.listEquals([
424 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, 422 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT,
425 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], 423 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
426 _utf8ToCodepoints([0xed, 0xaf, 0xbf, 0xed, 0xb0, 0x80]), 424 utf8ToCodepoints([0xed, 0xaf, 0xbf, 0xed, 0xb0, 0x80]),
427 "U+DBFF U+DC00"); 425 "U+DBFF U+DC00");
428 Expect.listEquals([ 426 Expect.listEquals([
429 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, 427 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT,
430 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], 428 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
431 _utf8ToCodepoints([0xed, 0xaf, 0xbf, 0xed, 0xbf, 0xbf]), 429 utf8ToCodepoints([0xed, 0xaf, 0xbf, 0xed, 0xbf, 0xbf]),
432 "U+DBFF U+DFFF"); 430 "U+DBFF U+DFFF");
433 431
434 // Other illegal code positions (???) 432 // Other illegal code positions (???)
435 Expect.listEquals([0xfffe], _utf8ToCodepoints([0xef, 0xbf, 0xbe]), 433 Expect.listEquals([0xfffe], utf8ToCodepoints([0xef, 0xbf, 0xbe]),
436 "U+FFFE"); 434 "U+FFFE");
437 Expect.listEquals([0xffff], _utf8ToCodepoints([0xef, 0xbf, 0xbf]), 435 Expect.listEquals([0xffff], utf8ToCodepoints([0xef, 0xbf, 0xbf]),
438 "U+FFFF"); 436 "U+FFFF");
439 } 437 }
440 438
441 void testUtf8BytesToString() { 439 void testUtf8BytesToString() {
442 Expect.stringEquals(testEnglishPhrase, 440 Expect.stringEquals(testEnglishPhrase,
443 decodeUtf8(testEnglishUtf8), "English"); 441 decodeUtf8(testEnglishUtf8), "English");
444 442
445 Expect.stringEquals(testDanishPhrase, 443 Expect.stringEquals(testDanishPhrase,
446 decodeUtf8(testDanishUtf8), "Danish"); 444 decodeUtf8(testDanishUtf8), "Danish");
447 445
(...skipping 20 matching lines...) Expand all
468 466
469 IterableUtf8Decoder kataDecoder = decodeUtf8AsIterable(testKatakanaUtf8); 467 IterableUtf8Decoder kataDecoder = decodeUtf8AsIterable(testKatakanaUtf8);
470 // get the first character 468 // get the first character
471 Expect.equals(testKatakanaPhrase.charCodes()[0], 469 Expect.equals(testKatakanaPhrase.charCodes()[0],
472 kataDecoder.iterator().next()); 470 kataDecoder.iterator().next());
473 // get the whole translation using the Iterable interface 471 // get the whole translation using the Iterable interface
474 Expect.stringEquals(testKatakanaPhrase, 472 Expect.stringEquals(testKatakanaPhrase,
475 new String.fromCharCodes(new List<int>.from(kataDecoder))); 473 new String.fromCharCodes(new List<int>.from(kataDecoder)));
476 } 474 }
477 } 475 }
OLDNEW
« no previous file with comments | « utils/tests/string_encoding/utf8_benchmarks.dart ('k') | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698