| OLD | NEW |
| 1 #!/usr/bin/env dart | 1 #!/usr/bin/env dart |
| 2 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 2 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
| 3 // for details. All rights reserved. Use of this source code is governed by a | 3 // for details. All rights reserved. Use of this source code is governed by a |
| 4 // BSD-style license that can be found in the LICENSE file. | 4 // BSD-style license that can be found in the LICENSE file. |
| 5 | 5 |
| 6 #library("utf8_tests"); | 6 #library("utf8_tests"); |
| 7 #import("dunit.dart"); | 7 #import("dunit.dart"); |
| 8 #import("../../string_encoding/unicode.dart"); | 8 #import("../../../lib/utf/utf.dart"); |
| 9 #import("../../string_encoding/unicode_core.dart"); | |
| 10 #source("../../string_encoding/utf8_impl.dart"); | |
| 11 | 9 |
| 12 void main() { | 10 void main() { |
| 13 TestSuite suite = new TestSuite(); | 11 TestSuite suite = new TestSuite(); |
| 14 suite.registerTestClass(new Utf8Tests()); | 12 suite.registerTestClass(new Utf8Tests()); |
| 15 suite.run(); | 13 suite.run(); |
| 16 } | 14 } |
| 17 | 15 |
| 18 class Utf8Tests extends TestClass { | 16 class Utf8Tests extends TestClass { |
| 19 static final String testEnglishPhrase = | 17 static final String testEnglishPhrase = |
| 20 "The quick brown fox jumps over the lazy dog."; | 18 "The quick brown fox jumps over the lazy dog."; |
| (...skipping 125 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 146 | 144 |
| 147 Expect.listEquals(testGreekUtf8, encodeUtf8(testGreekPhrase), | 145 Expect.listEquals(testGreekUtf8, encodeUtf8(testGreekPhrase), |
| 148 "Greek to utf8"); | 146 "Greek to utf8"); |
| 149 | 147 |
| 150 Expect.listEquals(testKatakanaUtf8, encodeUtf8(testKatakanaPhrase), | 148 Expect.listEquals(testKatakanaUtf8, encodeUtf8(testKatakanaPhrase), |
| 151 "Katakana to utf8"); | 149 "Katakana to utf8"); |
| 152 } | 150 } |
| 153 | 151 |
| 154 void testUtf8bytesToCodepoints() { | 152 void testUtf8bytesToCodepoints() { |
| 155 Expect.listEquals([954, 972, 963, 956, 949], | 153 Expect.listEquals([954, 972, 963, 956, 949], |
| 156 _utf8ToCodepoints([0xce, 0xba, 0xcf, 0x8c, 0xcf, | 154 utf8ToCodepoints([0xce, 0xba, 0xcf, 0x8c, 0xcf, |
| 157 0x83, 0xce, 0xbc, 0xce, 0xb5]), "κόσμε"); | 155 0x83, 0xce, 0xbc, 0xce, 0xb5]), "κόσμε"); |
| 158 | 156 |
| 159 // boundary conditions: First possible sequence of a certain length | 157 // boundary conditions: First possible sequence of a certain length |
| 160 Expect.listEquals([], _utf8ToCodepoints([]), "no input"); | 158 Expect.listEquals([], utf8ToCodepoints([]), "no input"); |
| 161 Expect.listEquals([0x0], _utf8ToCodepoints([0x0]), "0"); | 159 Expect.listEquals([0x0], utf8ToCodepoints([0x0]), "0"); |
| 162 Expect.listEquals([0x80], _utf8ToCodepoints([0xc2, 0x80]), "80"); | 160 Expect.listEquals([0x80], utf8ToCodepoints([0xc2, 0x80]), "80"); |
| 163 Expect.listEquals([0x800], | 161 Expect.listEquals([0x800], |
| 164 _utf8ToCodepoints([0xe0, 0xa0, 0x80]), "800"); | 162 utf8ToCodepoints([0xe0, 0xa0, 0x80]), "800"); |
| 165 Expect.listEquals([0x10000], | 163 Expect.listEquals([0x10000], |
| 166 _utf8ToCodepoints([0xf0, 0x90, 0x80, 0x80]), "10000"); | 164 utf8ToCodepoints([0xf0, 0x90, 0x80, 0x80]), "10000"); |
| 167 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | 165 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], |
| 168 _utf8ToCodepoints([0xf8, 0x88, 0x80, 0x80, 0x80]), "200000"); | 166 utf8ToCodepoints([0xf8, 0x88, 0x80, 0x80, 0x80]), "200000"); |
| 169 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | 167 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], |
| 170 _utf8ToCodepoints([0xfc, 0x84, 0x80, 0x80, 0x80, 0x80]), | 168 utf8ToCodepoints([0xfc, 0x84, 0x80, 0x80, 0x80, 0x80]), |
| 171 "4000000"); | 169 "4000000"); |
| 172 | 170 |
| 173 // boundary conditions: Last possible sequence of a certain length | 171 // boundary conditions: Last possible sequence of a certain length |
| 174 Expect.listEquals([0x7f], _utf8ToCodepoints([0x7f]), "7f"); | 172 Expect.listEquals([0x7f], utf8ToCodepoints([0x7f]), "7f"); |
| 175 Expect.listEquals([0x7ff], _utf8ToCodepoints([0xdf, 0xbf]), "7ff"); | 173 Expect.listEquals([0x7ff], utf8ToCodepoints([0xdf, 0xbf]), "7ff"); |
| 176 Expect.listEquals([0xffff], | 174 Expect.listEquals([0xffff], |
| 177 _utf8ToCodepoints([0xef, 0xbf, 0xbf]), "ffff"); | 175 utf8ToCodepoints([0xef, 0xbf, 0xbf]), "ffff"); |
| 178 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | 176 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], |
| 179 _utf8ToCodepoints([0xf7, 0xbf, 0xbf, 0xbf]), "1fffff"); | 177 utf8ToCodepoints([0xf7, 0xbf, 0xbf, 0xbf]), "1fffff"); |
| 180 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | 178 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], |
| 181 _utf8ToCodepoints([0xfb, 0xbf, 0xbf, 0xbf, 0xbf]), "3ffffff"); | 179 utf8ToCodepoints([0xfb, 0xbf, 0xbf, 0xbf, 0xbf]), "3ffffff"); |
| 182 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | 180 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], |
| 183 _utf8ToCodepoints([0xfd, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf]), | 181 utf8ToCodepoints([0xfd, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf]), |
| 184 "4000000"); | 182 "4000000"); |
| 185 | 183 |
| 186 // other boundary conditions | 184 // other boundary conditions |
| 187 Expect.listEquals([0xd7ff], | 185 Expect.listEquals([0xd7ff], |
| 188 _utf8ToCodepoints([0xed, 0x9f, 0xbf]), "d7ff"); | 186 utf8ToCodepoints([0xed, 0x9f, 0xbf]), "d7ff"); |
| 189 Expect.listEquals([0xe000], | 187 Expect.listEquals([0xe000], |
| 190 _utf8ToCodepoints([0xee, 0x80, 0x80]), "e000"); | 188 utf8ToCodepoints([0xee, 0x80, 0x80]), "e000"); |
| 191 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | 189 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], |
| 192 _utf8ToCodepoints([0xef, 0xbf, 0xbd]), "fffd"); | 190 utf8ToCodepoints([0xef, 0xbf, 0xbd]), "fffd"); |
| 193 Expect.listEquals([0x10ffff], | 191 Expect.listEquals([0x10ffff], |
| 194 _utf8ToCodepoints([0xf4, 0x8f, 0xbf, 0xbf]), "10ffff"); | 192 utf8ToCodepoints([0xf4, 0x8f, 0xbf, 0xbf]), "10ffff"); |
| 195 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | 193 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], |
| 196 _utf8ToCodepoints([0xf4, 0x90, 0x80, 0x80]), "110000"); | 194 utf8ToCodepoints([0xf4, 0x90, 0x80, 0x80]), "110000"); |
| 197 | 195 |
| 198 // unexpected continuation bytes | 196 // unexpected continuation bytes |
| 199 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | 197 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], |
| 200 _utf8ToCodepoints([0x80]), "80 => replacement character"); | 198 utf8ToCodepoints([0x80]), "80 => replacement character"); |
| 201 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | 199 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], |
| 202 _utf8ToCodepoints([0xbf]), "bf => replacement character"); | 200 utf8ToCodepoints([0xbf]), "bf => replacement character"); |
| 203 | 201 |
| 204 List<int> allContinuationBytes = <int>[]; | 202 List<int> allContinuationBytes = <int>[]; |
| 205 List<int> matchingReplacementChars = <int>[]; | 203 List<int> matchingReplacementChars = <int>[]; |
| 206 for (int i = 0x80; i < 0xc0; i++) { | 204 for (int i = 0x80; i < 0xc0; i++) { |
| 207 allContinuationBytes.add(i); | 205 allContinuationBytes.add(i); |
| 208 matchingReplacementChars.add(UNICODE_REPLACEMENT_CHARACTER_CODEPOINT); | 206 matchingReplacementChars.add(UNICODE_REPLACEMENT_CHARACTER_CODEPOINT); |
| 209 } | 207 } |
| 210 Expect.listEquals(matchingReplacementChars, | 208 Expect.listEquals(matchingReplacementChars, |
| 211 _utf8ToCodepoints(allContinuationBytes), | 209 utf8ToCodepoints(allContinuationBytes), |
| 212 "80 - bf => replacement character x 64"); | 210 "80 - bf => replacement character x 64"); |
| 213 | 211 |
| 214 List<int> allFirstTwoByteSeq = <int>[]; | 212 List<int> allFirstTwoByteSeq = <int>[]; |
| 215 matchingReplacementChars = <int>[]; | 213 matchingReplacementChars = <int>[]; |
| 216 for (int i = 0xc0; i < 0xe0; i++) { | 214 for (int i = 0xc0; i < 0xe0; i++) { |
| 217 allFirstTwoByteSeq.addAll([i, 0x20]); | 215 allFirstTwoByteSeq.addAll([i, 0x20]); |
| 218 matchingReplacementChars.addAll( | 216 matchingReplacementChars.addAll( |
| 219 [UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]); | 217 [UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]); |
| 220 } | 218 } |
| 221 Expect.listEquals(matchingReplacementChars, | 219 Expect.listEquals(matchingReplacementChars, |
| 222 _utf8ToCodepoints(allFirstTwoByteSeq), | 220 utf8ToCodepoints(allFirstTwoByteSeq), |
| 223 "c0 - df + space => replacement character + space x 32"); | 221 "c0 - df + space => replacement character + space x 32"); |
| 224 | 222 |
| 225 List<int> allFirstThreeByteSeq = <int>[]; | 223 List<int> allFirstThreeByteSeq = <int>[]; |
| 226 matchingReplacementChars = <int>[]; | 224 matchingReplacementChars = <int>[]; |
| 227 for (int i = 0xe0; i < 0xf0; i++) { | 225 for (int i = 0xe0; i < 0xf0; i++) { |
| 228 allFirstThreeByteSeq.addAll([i, 0x20]); | 226 allFirstThreeByteSeq.addAll([i, 0x20]); |
| 229 matchingReplacementChars.addAll( | 227 matchingReplacementChars.addAll( |
| 230 [UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]); | 228 [UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]); |
| 231 } | 229 } |
| 232 Expect.listEquals(matchingReplacementChars, | 230 Expect.listEquals(matchingReplacementChars, |
| 233 _utf8ToCodepoints(allFirstThreeByteSeq), | 231 utf8ToCodepoints(allFirstThreeByteSeq), |
| 234 "e0 - ef + space => replacement character x 16"); | 232 "e0 - ef + space => replacement character x 16"); |
| 235 | 233 |
| 236 List<int> allFirstFourByteSeq = <int>[]; | 234 List<int> allFirstFourByteSeq = <int>[]; |
| 237 matchingReplacementChars = <int>[]; | 235 matchingReplacementChars = <int>[]; |
| 238 for (int i = 0xf0; i < 0xf8; i++) { | 236 for (int i = 0xf0; i < 0xf8; i++) { |
| 239 allFirstFourByteSeq.addAll([i, 0x20]); | 237 allFirstFourByteSeq.addAll([i, 0x20]); |
| 240 matchingReplacementChars.addAll( | 238 matchingReplacementChars.addAll( |
| 241 [UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]); | 239 [UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]); |
| 242 } | 240 } |
| 243 Expect.listEquals(matchingReplacementChars, | 241 Expect.listEquals(matchingReplacementChars, |
| 244 _utf8ToCodepoints(allFirstFourByteSeq), | 242 utf8ToCodepoints(allFirstFourByteSeq), |
| 245 "f0 - f7 + space => replacement character x 8"); | 243 "f0 - f7 + space => replacement character x 8"); |
| 246 | 244 |
| 247 List<int> allFirstFiveByteSeq = <int>[]; | 245 List<int> allFirstFiveByteSeq = <int>[]; |
| 248 matchingReplacementChars = <int>[]; | 246 matchingReplacementChars = <int>[]; |
| 249 for (int i = 0xf8; i < 0xfc; i++) { | 247 for (int i = 0xf8; i < 0xfc; i++) { |
| 250 allFirstFiveByteSeq.addAll([i, 0x20]); | 248 allFirstFiveByteSeq.addAll([i, 0x20]); |
| 251 matchingReplacementChars.addAll( | 249 matchingReplacementChars.addAll( |
| 252 [UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]); | 250 [UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]); |
| 253 } | 251 } |
| 254 Expect.listEquals(matchingReplacementChars, | 252 Expect.listEquals(matchingReplacementChars, |
| 255 _utf8ToCodepoints(allFirstFiveByteSeq), | 253 utf8ToCodepoints(allFirstFiveByteSeq), |
| 256 "f8 - fb + space => replacement character x 4"); | 254 "f8 - fb + space => replacement character x 4"); |
| 257 | 255 |
| 258 List<int> allFirstSixByteSeq = <int>[]; | 256 List<int> allFirstSixByteSeq = <int>[]; |
| 259 matchingReplacementChars = <int>[]; | 257 matchingReplacementChars = <int>[]; |
| 260 for (int i = 0xfc; i < 0xfe; i++) { | 258 for (int i = 0xfc; i < 0xfe; i++) { |
| 261 allFirstSixByteSeq.addAll([i, 0x20]); | 259 allFirstSixByteSeq.addAll([i, 0x20]); |
| 262 matchingReplacementChars.addAll( | 260 matchingReplacementChars.addAll( |
| 263 [UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]); | 261 [UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]); |
| 264 } | 262 } |
| 265 Expect.listEquals(matchingReplacementChars, | 263 Expect.listEquals(matchingReplacementChars, |
| 266 _utf8ToCodepoints(allFirstSixByteSeq), | 264 utf8ToCodepoints(allFirstSixByteSeq), |
| 267 "fc - fd + space => replacement character x 2"); | 265 "fc - fd + space => replacement character x 2"); |
| 268 | 266 |
| 269 // Sequences with last continuation byte missing | 267 // Sequences with last continuation byte missing |
| 270 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | 268 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], |
| 271 _utf8ToCodepoints([0xc2]), | 269 utf8ToCodepoints([0xc2]), |
| 272 "2-byte sequence with last byte missing"); | 270 "2-byte sequence with last byte missing"); |
| 273 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | 271 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], |
| 274 _utf8ToCodepoints([0xe0, 0x80]), | 272 utf8ToCodepoints([0xe0, 0x80]), |
| 275 "3-byte sequence with last byte missing"); | 273 "3-byte sequence with last byte missing"); |
| 276 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | 274 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], |
| 277 _utf8ToCodepoints([0xf0, 0x80, 0x80]), | 275 utf8ToCodepoints([0xf0, 0x80, 0x80]), |
| 278 "4-byte sequence with last byte missing"); | 276 "4-byte sequence with last byte missing"); |
| 279 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | 277 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], |
| 280 _utf8ToCodepoints([0xf8, 0x88, 0x80, 0x80]), | 278 utf8ToCodepoints([0xf8, 0x88, 0x80, 0x80]), |
| 281 "5-byte sequence with last byte missing"); | 279 "5-byte sequence with last byte missing"); |
| 282 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | 280 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], |
| 283 _utf8ToCodepoints([0xfc, 0x80, 0x80, 0x80, 0x80]), | 281 utf8ToCodepoints([0xfc, 0x80, 0x80, 0x80, 0x80]), |
| 284 "6-byte sequence with last byte missing"); | 282 "6-byte sequence with last byte missing"); |
| 285 | 283 |
| 286 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | 284 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], |
| 287 _utf8ToCodepoints([0xdf]), | 285 utf8ToCodepoints([0xdf]), |
| 288 "2-byte sequence with last byte missing (hi)"); | 286 "2-byte sequence with last byte missing (hi)"); |
| 289 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | 287 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], |
| 290 _utf8ToCodepoints([0xef, 0xbf]), | 288 utf8ToCodepoints([0xef, 0xbf]), |
| 291 "3-byte sequence with last byte missing (hi)"); | 289 "3-byte sequence with last byte missing (hi)"); |
| 292 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | 290 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], |
| 293 _utf8ToCodepoints([0xf7, 0xbf, 0xbf]), | 291 utf8ToCodepoints([0xf7, 0xbf, 0xbf]), |
| 294 "4-byte sequence with last byte missing (hi)"); | 292 "4-byte sequence with last byte missing (hi)"); |
| 295 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | 293 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], |
| 296 _utf8ToCodepoints([0xfb, 0xbf, 0xbf, 0xbf]), | 294 utf8ToCodepoints([0xfb, 0xbf, 0xbf, 0xbf]), |
| 297 "5-byte sequence with last byte missing (hi)"); | 295 "5-byte sequence with last byte missing (hi)"); |
| 298 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | 296 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], |
| 299 _utf8ToCodepoints([0xfd, 0xbf, 0xbf, 0xbf, 0xbf]), | 297 utf8ToCodepoints([0xfd, 0xbf, 0xbf, 0xbf, 0xbf]), |
| 300 "6-byte sequence with last byte missing (hi)"); | 298 "6-byte sequence with last byte missing (hi)"); |
| 301 | 299 |
| 302 // Concatenation of incomplete sequences | 300 // Concatenation of incomplete sequences |
| 303 Expect.listEquals( | 301 Expect.listEquals( |
| 304 [ UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, | 302 [ UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, |
| 305 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, | 303 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, |
| 306 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, | 304 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, |
| 307 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, | 305 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, |
| 308 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, | 306 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, |
| 309 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, | 307 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, |
| 310 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, | 308 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, |
| 311 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, | 309 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, |
| 312 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, | 310 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, |
| 313 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT ], | 311 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT ], |
| 314 _utf8ToCodepoints( | 312 utf8ToCodepoints( |
| 315 [ 0xc2, | 313 [ 0xc2, |
| 316 0xe0, 0x80, | 314 0xe0, 0x80, |
| 317 0xf0, 0x80, 0x80, | 315 0xf0, 0x80, 0x80, |
| 318 0xf8, 0x88, 0x80, 0x80, | 316 0xf8, 0x88, 0x80, 0x80, |
| 319 0xfc, 0x80, 0x80, 0x80, 0x80, | 317 0xfc, 0x80, 0x80, 0x80, 0x80, |
| 320 0xdf, | 318 0xdf, |
| 321 0xef, 0xbf, | 319 0xef, 0xbf, |
| 322 0xf7, 0xbf, 0xbf, | 320 0xf7, 0xbf, 0xbf, |
| 323 0xfb, 0xbf, 0xbf, 0xbf, | 321 0xfb, 0xbf, 0xbf, 0xbf, |
| 324 0xfd, 0xbf, 0xbf, 0xbf, 0xbf ]), | 322 0xfd, 0xbf, 0xbf, 0xbf, 0xbf ]), |
| 325 "Concatenation of incomplete sequences"); | 323 "Concatenation of incomplete sequences"); |
| 326 | 324 |
| 327 // Impossible bytes | 325 // Impossible bytes |
| 328 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | 326 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], |
| 329 _utf8ToCodepoints([0xfe]), "fe"); | 327 utf8ToCodepoints([0xfe]), "fe"); |
| 330 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | 328 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], |
| 331 _utf8ToCodepoints([0xff]), "ff"); | 329 utf8ToCodepoints([0xff]), "ff"); |
| 332 Expect.listEquals([ | 330 Expect.listEquals([ |
| 333 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, | 331 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, |
| 334 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, | 332 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, |
| 335 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, | 333 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, |
| 336 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | 334 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], |
| 337 _utf8ToCodepoints([0xfe, 0xfe, 0xff, 0xff]), "fe fe ff ff"); | 335 utf8ToCodepoints([0xfe, 0xfe, 0xff, 0xff]), "fe fe ff ff"); |
| 338 | 336 |
| 339 // Overlong sequences | 337 // Overlong sequences |
| 340 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | 338 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], |
| 341 _utf8ToCodepoints([0xc0, 0xaf]), "c0 af"); | 339 utf8ToCodepoints([0xc0, 0xaf]), "c0 af"); |
| 342 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | 340 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], |
| 343 _utf8ToCodepoints([0xe0, 0x80, 0xaf]), "e0 80 af"); | 341 utf8ToCodepoints([0xe0, 0x80, 0xaf]), "e0 80 af"); |
| 344 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | 342 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], |
| 345 _utf8ToCodepoints([0xf0, 0x80, 0x80, 0xaf]), "f0 80 80 af"); | 343 utf8ToCodepoints([0xf0, 0x80, 0x80, 0xaf]), "f0 80 80 af"); |
| 346 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | 344 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], |
| 347 _utf8ToCodepoints([0xf8, 0x80, 0x80, 0x80, 0xaf]), "f8 80 80 80 af"); | 345 utf8ToCodepoints([0xf8, 0x80, 0x80, 0x80, 0xaf]), "f8 80 80 80 af"); |
| 348 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | 346 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], |
| 349 _utf8ToCodepoints([0xfc, 0x80, 0x80, 0x80, 0x80, 0xaf]), | 347 utf8ToCodepoints([0xfc, 0x80, 0x80, 0x80, 0x80, 0xaf]), |
| 350 "fc 80 80 80 80 af"); | 348 "fc 80 80 80 80 af"); |
| 351 | 349 |
| 352 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | 350 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], |
| 353 _utf8ToCodepoints([0xc1, 0xbf]), "c1 bf"); | 351 utf8ToCodepoints([0xc1, 0xbf]), "c1 bf"); |
| 354 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | 352 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], |
| 355 _utf8ToCodepoints([0xe0, 0x9f, 0xbf]), "e0 9f bf"); | 353 utf8ToCodepoints([0xe0, 0x9f, 0xbf]), "e0 9f bf"); |
| 356 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | 354 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], |
| 357 _utf8ToCodepoints([0xf0, 0x8f, 0xbf, 0xbf]), "f0 8f bf bf"); | 355 utf8ToCodepoints([0xf0, 0x8f, 0xbf, 0xbf]), "f0 8f bf bf"); |
| 358 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | 356 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], |
| 359 _utf8ToCodepoints([0xf8, 0x87, 0xbf, 0xbf, 0xbf]), "f8 87 bf bf bf"); | 357 utf8ToCodepoints([0xf8, 0x87, 0xbf, 0xbf, 0xbf]), "f8 87 bf bf bf"); |
| 360 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | 358 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], |
| 361 _utf8ToCodepoints([0xfc, 0x83, 0xbf, 0xbf, 0xbf, 0xbf]), | 359 utf8ToCodepoints([0xfc, 0x83, 0xbf, 0xbf, 0xbf, 0xbf]), |
| 362 "fc 83 bf bf bf bf"); | 360 "fc 83 bf bf bf bf"); |
| 363 | 361 |
| 364 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | 362 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], |
| 365 _utf8ToCodepoints([0xc0, 0x80]), "c0 80"); | 363 utf8ToCodepoints([0xc0, 0x80]), "c0 80"); |
| 366 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | 364 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], |
| 367 _utf8ToCodepoints([0xe0, 0x80, 0x80]), "e0 80 80"); | 365 utf8ToCodepoints([0xe0, 0x80, 0x80]), "e0 80 80"); |
| 368 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | 366 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], |
| 369 _utf8ToCodepoints([0xf0, 0x80, 0x80, 0x80]), "f0 80 80 80"); | 367 utf8ToCodepoints([0xf0, 0x80, 0x80, 0x80]), "f0 80 80 80"); |
| 370 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | 368 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], |
| 371 _utf8ToCodepoints([0xf8, 0x80, 0x80, 0x80, 0x80]), "f8 80 80 80 80"); | 369 utf8ToCodepoints([0xf8, 0x80, 0x80, 0x80, 0x80]), "f8 80 80 80 80"); |
| 372 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | 370 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], |
| 373 _utf8ToCodepoints([0xfc, 0x80, 0x80, 0x80, 0x80, 0x80]), | 371 utf8ToCodepoints([0xfc, 0x80, 0x80, 0x80, 0x80, 0x80]), |
| 374 "fc 80 80 80 80 80"); | 372 "fc 80 80 80 80 80"); |
| 375 | 373 |
| 376 // Illegal code positions | 374 // Illegal code positions |
| 377 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | 375 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], |
| 378 _utf8ToCodepoints([0xed, 0xa0, 0x80]), "U+D800"); | 376 utf8ToCodepoints([0xed, 0xa0, 0x80]), "U+D800"); |
| 379 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | 377 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], |
| 380 _utf8ToCodepoints([0xed, 0xad, 0xbf]), "U+DB7F"); | 378 utf8ToCodepoints([0xed, 0xad, 0xbf]), "U+DB7F"); |
| 381 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | 379 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], |
| 382 _utf8ToCodepoints([0xed, 0xae, 0x80]), "U+DB80"); | 380 utf8ToCodepoints([0xed, 0xae, 0x80]), "U+DB80"); |
| 383 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | 381 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], |
| 384 _utf8ToCodepoints([0xed, 0xaf, 0xbf]), "U+DBFF"); | 382 utf8ToCodepoints([0xed, 0xaf, 0xbf]), "U+DBFF"); |
| 385 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | 383 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], |
| 386 _utf8ToCodepoints([0xed, 0xb0, 0x80]), "U+DC00"); | 384 utf8ToCodepoints([0xed, 0xb0, 0x80]), "U+DC00"); |
| 387 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | 385 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], |
| 388 _utf8ToCodepoints([0xed, 0xbe, 0x80]), "U+DF80"); | 386 utf8ToCodepoints([0xed, 0xbe, 0x80]), "U+DF80"); |
| 389 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | 387 Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], |
| 390 _utf8ToCodepoints([0xed, 0xbf, 0xbf]), "U+DFFF"); | 388 utf8ToCodepoints([0xed, 0xbf, 0xbf]), "U+DFFF"); |
| 391 | 389 |
| 392 // Paired UTF-16 surrogates | 390 // Paired UTF-16 surrogates |
| 393 Expect.listEquals([ | 391 Expect.listEquals([ |
| 394 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, | 392 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, |
| 395 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | 393 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], |
| 396 _utf8ToCodepoints([0xed, 0xa0, 0x80, 0xed, 0xb0, 0x80]), | 394 utf8ToCodepoints([0xed, 0xa0, 0x80, 0xed, 0xb0, 0x80]), |
| 397 "U+D800 U+DC00"); | 395 "U+D800 U+DC00"); |
| 398 Expect.listEquals([ | 396 Expect.listEquals([ |
| 399 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, | 397 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, |
| 400 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | 398 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], |
| 401 _utf8ToCodepoints([0xed, 0xa0, 0x80, 0xed, 0xbf, 0xbf]), | 399 utf8ToCodepoints([0xed, 0xa0, 0x80, 0xed, 0xbf, 0xbf]), |
| 402 "U+D800 U+DFFF"); | 400 "U+D800 U+DFFF"); |
| 403 Expect.listEquals([ | 401 Expect.listEquals([ |
| 404 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, | 402 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, |
| 405 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | 403 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], |
| 406 _utf8ToCodepoints([0xed, 0xad, 0xbf, 0xed, 0xb0, 0x80]), | 404 utf8ToCodepoints([0xed, 0xad, 0xbf, 0xed, 0xb0, 0x80]), |
| 407 "U+DB7F U+DC00"); | 405 "U+DB7F U+DC00"); |
| 408 Expect.listEquals([ | 406 Expect.listEquals([ |
| 409 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, | 407 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, |
| 410 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | 408 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], |
| 411 _utf8ToCodepoints([0xed, 0xad, 0xbf, 0xed, 0xbf, 0xbf]), | 409 utf8ToCodepoints([0xed, 0xad, 0xbf, 0xed, 0xbf, 0xbf]), |
| 412 "U+DB7F U+DFFF"); | 410 "U+DB7F U+DFFF"); |
| 413 Expect.listEquals([ | 411 Expect.listEquals([ |
| 414 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, | 412 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, |
| 415 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | 413 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], |
| 416 _utf8ToCodepoints([0xed, 0xae, 0x80, 0xed, 0xb0, 0x80]), | 414 utf8ToCodepoints([0xed, 0xae, 0x80, 0xed, 0xb0, 0x80]), |
| 417 "U+DB80 U+DC00"); | 415 "U+DB80 U+DC00"); |
| 418 Expect.listEquals([ | 416 Expect.listEquals([ |
| 419 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, | 417 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, |
| 420 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | 418 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], |
| 421 _utf8ToCodepoints([0xed, 0xae, 0x80, 0xed, 0xbf, 0xbf]), | 419 utf8ToCodepoints([0xed, 0xae, 0x80, 0xed, 0xbf, 0xbf]), |
| 422 "U+DB80 U+DFFF"); | 420 "U+DB80 U+DFFF"); |
| 423 Expect.listEquals([ | 421 Expect.listEquals([ |
| 424 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, | 422 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, |
| 425 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | 423 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], |
| 426 _utf8ToCodepoints([0xed, 0xaf, 0xbf, 0xed, 0xb0, 0x80]), | 424 utf8ToCodepoints([0xed, 0xaf, 0xbf, 0xed, 0xb0, 0x80]), |
| 427 "U+DBFF U+DC00"); | 425 "U+DBFF U+DC00"); |
| 428 Expect.listEquals([ | 426 Expect.listEquals([ |
| 429 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, | 427 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, |
| 430 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], | 428 UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], |
| 431 _utf8ToCodepoints([0xed, 0xaf, 0xbf, 0xed, 0xbf, 0xbf]), | 429 utf8ToCodepoints([0xed, 0xaf, 0xbf, 0xed, 0xbf, 0xbf]), |
| 432 "U+DBFF U+DFFF"); | 430 "U+DBFF U+DFFF"); |
| 433 | 431 |
| 434 // Other illegal code positions (???) | 432 // Other illegal code positions (???) |
| 435 Expect.listEquals([0xfffe], _utf8ToCodepoints([0xef, 0xbf, 0xbe]), | 433 Expect.listEquals([0xfffe], utf8ToCodepoints([0xef, 0xbf, 0xbe]), |
| 436 "U+FFFE"); | 434 "U+FFFE"); |
| 437 Expect.listEquals([0xffff], _utf8ToCodepoints([0xef, 0xbf, 0xbf]), | 435 Expect.listEquals([0xffff], utf8ToCodepoints([0xef, 0xbf, 0xbf]), |
| 438 "U+FFFF"); | 436 "U+FFFF"); |
| 439 } | 437 } |
| 440 | 438 |
| 441 void testUtf8BytesToString() { | 439 void testUtf8BytesToString() { |
| 442 Expect.stringEquals(testEnglishPhrase, | 440 Expect.stringEquals(testEnglishPhrase, |
| 443 decodeUtf8(testEnglishUtf8), "English"); | 441 decodeUtf8(testEnglishUtf8), "English"); |
| 444 | 442 |
| 445 Expect.stringEquals(testDanishPhrase, | 443 Expect.stringEquals(testDanishPhrase, |
| 446 decodeUtf8(testDanishUtf8), "Danish"); | 444 decodeUtf8(testDanishUtf8), "Danish"); |
| 447 | 445 |
| (...skipping 20 matching lines...) Expand all Loading... |
| 468 | 466 |
| 469 IterableUtf8Decoder kataDecoder = decodeUtf8AsIterable(testKatakanaUtf8); | 467 IterableUtf8Decoder kataDecoder = decodeUtf8AsIterable(testKatakanaUtf8); |
| 470 // get the first character | 468 // get the first character |
| 471 Expect.equals(testKatakanaPhrase.charCodes()[0], | 469 Expect.equals(testKatakanaPhrase.charCodes()[0], |
| 472 kataDecoder.iterator().next()); | 470 kataDecoder.iterator().next()); |
| 473 // get the whole translation using the Iterable interface | 471 // get the whole translation using the Iterable interface |
| 474 Expect.stringEquals(testKatakanaPhrase, | 472 Expect.stringEquals(testKatakanaPhrase, |
| 475 new String.fromCharCodes(new List<int>.from(kataDecoder))); | 473 new String.fromCharCodes(new List<int>.from(kataDecoder))); |
| 476 } | 474 } |
| 477 } | 475 } |
| OLD | NEW |