String encoding utility methods and tests for Unicode, UTF-8, -16 and -32.

utils/string_encoding/UnicodeCore.dart

Index: utils/string_encoding/UnicodeCore.dart
diff --git a/utils/string_encoding/UnicodeCore.dart b/utils/string_encoding/UnicodeCore.dart
new file mode 100644
index 0000000000000000000000000000000000000000..44da39d897a6f7ad5a9f877468f5fad6941486fe
--- /dev/null
+++ b/utils/string_encoding/UnicodeCore.dart
@@ -0,0 +1,97 @@
+// Copyright (c) 2011, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+#library("UnicodeCore");
+
+/*
+ * Test for presence of bug related to the use of UTF-16 code units for
+ * Dart compiled to JS.
+ */
+bool _test16BitCodeUnit = null;
+// TODO is16BitCodeUnit() is used to work around a bug with frog/dartc
+// (http://code.google.com/p/dart/issues/detail?id=1357). Consider
+// removing after this issue is resolved.
+bool is16BitCodeUnit() {
+  if(_test16BitCodeUnit == null) {
+    _test16BitCodeUnit = (new String.fromCharCodes([0x1D11E])) ==
+        (new String.fromCharCodes([0xD11E]));
+  }
+  return _test16BitCodeUnit;
+}
+
+
+/**
+ * Invalid codepoints or encodings may be substituted with the value U+fffd.
+ */
+final int REPLACEMENT_CHARACTER_CODEPOINT = 0xfffd;
+
+/**
+ * Encode code points as UTF16 code units.
+ */
+List<int> codepointsToUtf16CodeUnits(List<int> codepoints,
+    [int start = 0, int length = -1]) {
+  List<int> codeUnitsBuffer = <int>[];
+  int end = length >= 0 ? Math.min(codepoints.length, start + length) :
+      codepoints.length;
+  int i = start;
+  while (i < end) {
+    int value = codepoints[i++];
+    if ((value >= 0 && value <0xd800) || (value >= 0xe000 && value <0x10000)) {
+      codeUnitsBuffer.add(value);
+    } else if (value >= 0x10000 && value < 0x110000){
+      int base = value - 0x10000;
+      codeUnitsBuffer.add(0xd800 + ((base & 0xffc00) >> 10));
+      codeUnitsBuffer.add(0xdc00 + (base & 0x3ff));
+    } else {
+      codeUnitsBuffer.add(REPLACEMENT_CHARACTER_CODEPOINT);
+    }
+  }
+  return codeUnitsBuffer;
+}
+
+/**
+ * Decodes the utf16 codeunits to codepoints.
+ */
+List<int> utf16CodeUnitsToCodepoints(List<int> utf16CodeUnits,
+    [int start = 0, int length = - 1]) {
+  List<int> codepointBuffer = <int>[];
+
+  int end = length >= 0 ?
+      Math.min(utf16CodeUnits.length, start + length) :
+      utf16CodeUnits.length;
+
+  int i = start;
+  // skip the first entry if it is a BOM.
+  if(end > 0 && utf16CodeUnits[0] == 0xfeff) {
+    i++;
+  }
+  while (i < end) {
+    int value = utf16CodeUnits[i++];
+    if (value >= 0x0) {
+      if (value < 0xd800 || (value >= 0xe000 && value <= 0xffff)) {
+        // transfer directly
+        codepointBuffer.add(value);
+      } else if (value < 0xdc00 && i < end) {
+        // merge surrogate pair
+        value = (value - 0xd800) * 0x400;
+        int nextValue = utf16CodeUnits[i++];
+        if (nextValue >= 0xdc00 && nextValue < 0xe000) {
+          value += 0x10000 + (nextValue - 0xdc00);
+          codepointBuffer.add(value);
+        } else {
+          if (nextValue >= 0xd800 && nextValue < 0xdc00) i--;
+          codepointBuffer.add(REPLACEMENT_CHARACTER_CODEPOINT);
+          continue;
+        }
+      } else {
+        codepointBuffer.add(REPLACEMENT_CHARACTER_CODEPOINT);
+        continue;
+      }
+    } else {
+      codepointBuffer.add(REPLACEMENT_CHARACTER_CODEPOINT);
+      continue;
+    }
+  }
+  return codepointBuffer;
+}