String encoding utility methods and tests for Unicode, UTF-8, -16 and -32.

utils/string_encoding/UnicodeCore.dart

Index: utils/string_encoding/UnicodeCore.dart
diff --git a/utils/string_encoding/UnicodeCore.dart b/utils/string_encoding/UnicodeCore.dart
new file mode 100644
index 0000000000000000000000000000000000000000..3ca8cce05461b7e61a6d25f1bb51abedd46ea316
--- /dev/null
+++ b/utils/string_encoding/UnicodeCore.dart
@@ -0,0 +1,139 @@
+// Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+#library("UnicodeCore");
+
+/*
+ * Test for presence of bug related to the use of UTF-16 code units for
+ * Dart compiled to JS.
+ */
+bool _test16BitCodeUnit = null;
+// TODO is16BitCodeUnit() is used to work around a bug with frog/dartc
+// (http://code.google.com/p/dart/issues/detail?id=1357). Consider
+// removing after this issue is resolved.
+bool is16BitCodeUnit() {
+  if (_test16BitCodeUnit == null) {
+    _test16BitCodeUnit = (new String.fromCharCodes([0x1D11E])) ==
+        (new String.fromCharCodes([0xD11E]));
+  }
+  return _test16BitCodeUnit;
+}
+
+
+/**
+ * Invalid codepoints or encodings may be substituted with the value U+fffd.
+ */
+final int UNICODE_REPLACEMENT_CHARACTER_CODEPOINT = 0xfffd;
+final int UNICODE_BOM = 0xfeff;
+final int UNICODE_UTF_BOM_LO = 0xff;
+final int UNICODE_UTF_BOM_HI = 0xfe;
+
+final int UNICODE_BYTE_ZERO_MASK = 0xff;
+final int UNICODE_BYTE_ONE_MASK = 0xff00;
+final int UNICODE_VALID_RANGE_MAX = 0x10ffff;
+final int UNICODE_PLANE_ONE_MAX = 0xffff;
+final int UNICODE_UTF16_RESERVED_LO = 0xd800;
+final int UNICODE_UTF16_RESERVED_HI = 0xdfff;
+final int UNICODE_UTF16_OFFSET = 0x10000;
+final int UNICODE_UTF16_SURROGATE_UNIT_0_BASE = 0xd800;
+final int UNICODE_UTF16_SURROGATE_UNIT_1_BASE = 0xdc00;
+final int UNICODE_UTF16_HI_MASK = 0xffc00;
+final int UNICODE_UTF16_LO_MASK = 0x3ff;
+
+/**
+ * Encode code points as UTF16 code units.

[Søren Gjesse, 2012/02/01 11:25:25]

Please be consistent with UTF16/utf16 in comments. Actually the "official" name
seems to be UTF-16.

[dcarlson, 2012/02/01 22:18:46]

Done. I went ahead and made sure all the variants match the spec.

+ */
+List<int> codepointsToUtf16CodeUnits(List<int> codepoints,
+    [int start = 0, int length = null]) {
+  if (!(start >= 0)) {
+    throw new IllegalArgumentException("start");
+  }
+
+  if (!(length == null || length >= 0)) {
+    throw new IllegalArgumentException("length");
+  }
+
+  int end = length != null ?
+      Math.min(codepoints.length, start + length) :
+      codepoints.length;
+
+  List<int> codeUnitsBuffer = <int>[];
+  int i = start;
+  while (i < end) {
+    int value = codepoints[i++];
+    if ((value >= 0 && value < UNICODE_UTF16_RESERVED_LO) || 
+        (value > UNICODE_UTF16_RESERVED_HI && value <= UNICODE_PLANE_ONE_MAX)) {
+      codeUnitsBuffer.add(value);
+    } else if (value > UNICODE_PLANE_ONE_MAX && 
+        value <= UNICODE_VALID_RANGE_MAX) {
+      int base = value - UNICODE_UTF16_OFFSET;
+      codeUnitsBuffer.add(UNICODE_UTF16_SURROGATE_UNIT_0_BASE +
+          ((base & UNICODE_UTF16_HI_MASK) >> 10));
+      codeUnitsBuffer.add(UNICODE_UTF16_SURROGATE_UNIT_1_BASE +
+          (base & UNICODE_UTF16_LO_MASK));
+    } else {
+      codeUnitsBuffer.add(UNICODE_REPLACEMENT_CHARACTER_CODEPOINT);
+    }
+  }
+  return codeUnitsBuffer;
+}
+
+/**
+ * Decodes the utf16 codeunits to codepoints.
+ */
+List<int> utf16CodeUnitsToCodepoints(List<int> utf16CodeUnits,
+    [int start = 0, int length = null]) {

[Søren Gjesse, 2012/02/01 11:25:25]

Normally we don't split argument lists like this. Won't

List<int> utf16CodeUnitsToCodepoints(
    List<int> utf16CodeUnits, [int start = 0, int length = null]) {

fit?

[dcarlson, 2012/02/01 22:18:46]

Not always (other lines), but done with a simple line wrap.

+  if (!(start >= 0)) {
+    throw new IllegalArgumentException("start");
+  }
+
+  if (!(length == null || length >= 0)) {
+    throw new IllegalArgumentException("length");
+  }
+
+  int end = length != null ?
+      Math.min(utf16CodeUnits.length, start + length) :
+      utf16CodeUnits.length;
+
+  List<int> codepointBuffer = <int>[];
+  int i = start;
+  // skip the first entry if it is a BOM.
+  if (end > 0 && utf16CodeUnits[0] == UNICODE_BOM) {
+    i++;
+  }
+  while (i < end) {
+    int value = utf16CodeUnits[i++];
+    if (value < 0) {
+      codepointBuffer.add(UNICODE_REPLACEMENT_CHARACTER_CODEPOINT);
+      continue;
+    }
+    if (value < UNICODE_UTF16_RESERVED_LO || 
+        (value > UNICODE_UTF16_RESERVED_HI &&
+        value <= UNICODE_PLANE_ONE_MAX)) {
+      // transfer directly
+      codepointBuffer.add(value);
+    } else if (value < UNICODE_UTF16_SURROGATE_UNIT_1_BASE && i < end) {
+      // merge surrogate pair
+      value = (value - UNICODE_UTF16_SURROGATE_UNIT_0_BASE) << 10;
+      int nextValue = utf16CodeUnits[i++];
+      if (nextValue >= UNICODE_UTF16_SURROGATE_UNIT_1_BASE && 
+          nextValue <= UNICODE_UTF16_RESERVED_HI) {
+        value += UNICODE_UTF16_OFFSET +
+            (nextValue - UNICODE_UTF16_SURROGATE_UNIT_1_BASE);
+        codepointBuffer.add(value);
+      } else {
+        if (nextValue >= UNICODE_UTF16_SURROGATE_UNIT_0_BASE &&
+           nextValue < UNICODE_UTF16_SURROGATE_UNIT_1_BASE) {
+          i--;
+        }
+        codepointBuffer.add(UNICODE_REPLACEMENT_CHARACTER_CODEPOINT);
+        continue;
+      }
+    } else {
+      codepointBuffer.add(UNICODE_REPLACEMENT_CHARACTER_CODEPOINT);
+      continue;
+    }
+  }
+  return codepointBuffer;
+}