String encoding utility methods and tests for Unicode, UTF-8, -16 and -32.

utils/tests/string_encoding/UnicodeCoreTests.dart

Index: utils/tests/string_encoding/UnicodeCoreTests.dart
diff --git a/utils/tests/string_encoding/UnicodeCoreTests.dart b/utils/tests/string_encoding/UnicodeCoreTests.dart
new file mode 100755
index 0000000000000000000000000000000000000000..adc8bc8bb39ce680079ab2d6fe000068d8880b82
--- /dev/null
+++ b/utils/tests/string_encoding/UnicodeCoreTests.dart
@@ -0,0 +1,103 @@
+#!/usr/bin/env dart
+// Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+#library("UnicodeCoreTests");
+#import("DUnit.dart");
+#import("../../string_encoding/UnicodeCore.dart");
+
+void main() {
+  TestSuite suite = new TestSuite();
+  suite.registerTestClass(new UnicodeCoreTests());
+  suite.run();
+}
+
+class UnicodeCoreTests extends TestClass {
+  void registerTests(TestSuite suite) {
+    register("testPlatform", testPlatform, suite);
+    register("testCodepointsToUtf16CodeUnits",
+        testCodepointsToUtf16CodeUnits, suite);
+    register("testUtf16bytesToCodepoints", testUtf16bytesToCodepoints, suite);
+  }
+
+  void testPlatform() {
+    Expect.isFalse(is16BitCodeUnit());
+  }
+
+  void testCodepointsToUtf16CodeUnits() {
+    // boundary conditions

[Søren Gjesse, 2012/02/01 11:25:25]

How about having a shared list of the "sane" test data:

... = [[0x0], "0"],
       [0xd800, 0xdc00], "10000",
       ...];

And reuse then in both tests.

[dcarlson, 2012/02/01 22:18:46]

Thanks for the suggestion -- I think this is clearer for the number of
datapoints included here.

+    Expect.listEquals([0x0], codepointsToUtf16CodeUnits([0x0]), "0");
+    Expect.listEquals([0xd800, 0xdc00],
+        codepointsToUtf16CodeUnits([0x10000]), "10000");
+
+    Expect.listEquals([0xffff],
+        codepointsToUtf16CodeUnits([0xffff]), "ffff");
+    Expect.listEquals([0xdbff, 0xdfff],
+        codepointsToUtf16CodeUnits([0x10ffff]), "10ffff");
+
+    Expect.listEquals([0xd7ff],
+        codepointsToUtf16CodeUnits([0xd7ff]), "d7ff");
+    Expect.listEquals([0xe000],
+        codepointsToUtf16CodeUnits([0xe000]), "e000");
+
+    Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
+        codepointsToUtf16CodeUnits([0xd800]), "d800");
+    Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
+        codepointsToUtf16CodeUnits([0xdfff]), "dfff");
+  }
+
+  void testUtf16bytesToCodepoints() {
+    // boundary conditions: First possible values
+    Expect.listEquals([0x0], utf16CodeUnitsToCodepoints([0x0]), "0");
+    Expect.listEquals([0x10000],
+        utf16CodeUnitsToCodepoints([0xd800, 0xdc00]), "10000");
+
+    // boundary conditions: Last possible sequence of a certain length
+    Expect.listEquals([0xffff],
+        utf16CodeUnitsToCodepoints([0xffff]), "ffff");
+    Expect.listEquals([0x10ffff],
+        utf16CodeUnitsToCodepoints([0xdbff, 0xdfff]), "10ffff");
+
+    // other boundary conditions
+    Expect.listEquals([0xd7ff],
+        utf16CodeUnitsToCodepoints([0xd7ff]), "d7ff");
+    Expect.listEquals([0xe000],
+        utf16CodeUnitsToCodepoints([0xe000]), "e000");
+
+    // unexpected continuation bytes
+    Expect.listEquals([0xfffd],
+        utf16CodeUnitsToCodepoints([0xdc00]),
+        "dc00 first unexpected continuation byte");
+    Expect.listEquals([0xfffd],
+        utf16CodeUnitsToCodepoints([0xdfff]),
+        "dfff last unexpected continuation byte");
+    Expect.listEquals([0xfffd],
+        utf16CodeUnitsToCodepoints([0xdc00]),
+        "1 unexpected continuation bytes");
+    Expect.listEquals([0xfffd, 0xfffd],
+        utf16CodeUnitsToCodepoints([0xdc00, 0xdc00]),
+        "2 unexpected continuation bytes");
+    Expect.listEquals([0xfffd, 0xfffd ,0xfffd],
+        utf16CodeUnitsToCodepoints([0xdc00, 0xdc00, 0xdc00]),
+        "3 unexpected continuation bytes");
+
+    // incomplete sequences
+    Expect.listEquals([0xfffd], utf16CodeUnitsToCodepoints([0xd800]),
+        "d800 last byte missing");
+    Expect.listEquals([0xfffd], utf16CodeUnitsToCodepoints([0xdbff]),
+        "dbff last byte missing");
+
+    // concatenation of incomplete sequences
+    Expect.listEquals([0xfffd, 0xfffd],
+        utf16CodeUnitsToCodepoints([0xd800, 0xdbff]),
+        "d800 dbff last byte missing");
+
+    // impossible bytes
+    Expect.listEquals([0xfffd], utf16CodeUnitsToCodepoints([0x110000]),
+        "110000 out of bounds");
+
+    // overlong sequences not possible in utf16 (nothing < x10000)
+    // illegal code positions d800-dfff not encodable (< x10000)
+  }
+}