String encoding utility methods and tests for Unicode, UTF-8, -16 and -32.

utils/string_encoding/Utf32.dart

+// Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+#library("UTF32");
+#import("UnicodeCore.dart");
+#import("Unicode.dart");
+
+/**
+ * Produce a String from a sequence of UTF32 encoded bytes.
+ */
+String decodeFromUtf32(List<int> bytes) =>
+    codepointsToString(_utf32ToCodePoints(bytes));
+
+/**
+ * Produce a String from a sequence of UTF32-BE encoded bytes.
+ */
+String decodeFromUtf32be(List<int> bytes) =>
+    codepointsToString(_utf32beToCodePoints(bytes));
+
+/**
+ * Produce a String from a sequence of UTF32-LE encoded bytes.
+ */
+String decodeFromUtf32le(List<int> bytes) =>
+    codepointsToString(_utf32leToCodePoints(bytes));
+
+/**
+ * Produce a sequence of UTF32 encoded bytes.
+ */
+List<int> encodeAsUtf32(String str, [bool writeBOM = true]) =>
+    encodeAsUtf32be(str, writeBOM);
+
+/**
+ * Produce a sequence of UTF32-BE encoded bytes.
+ */
+List<int> encodeAsUtf32be(String str, [bool writeBOM = false]) {
+  List<int> utf32CodeUnits = stringToCodepoints(str);
+  List<int> encoding = new List<int>(4 * utf32CodeUnits.length + 
+      (writeBOM ? 4 : 0));
+  int i = 0;
+  if (writeBOM) {
+    encoding[i++] = 0;
+    encoding[i++] = 0;
+    encoding[i++] = UNICODE_UTF_BOM_HI;
+    encoding[i++] = UNICODE_UTF_BOM_LO;
+  }
+  for (int unit in utf32CodeUnits) {
+    encoding[i++] = (unit >> 24) & UNICODE_BYTE_ZERO_MASK;
+    encoding[i++] = (unit >> 16) & UNICODE_BYTE_ZERO_MASK;
+    encoding[i++] = (unit >> 8) & UNICODE_BYTE_ZERO_MASK;
+    encoding[i++] = unit & UNICODE_BYTE_ZERO_MASK;
+  }
+  return encoding;
+}
+
+/**
+ * Produce a sequence of UTF32-LE encoded bytes.
+ */
+List<int> encodeAsUtf32le(String str, [bool writeBOM = false]) {
+  List<int> utf32CodeUnits = stringToCodepoints(str);
+  List<int> encoding = new List<int>(4 * utf32CodeUnits.length + 
+      (writeBOM ? 4 : 0));
+  int i = 0;
+  if (writeBOM) {
+    encoding[i++] = UNICODE_UTF_BOM_LO;
+    encoding[i++] = UNICODE_UTF_BOM_HI;
+    encoding[i++] = 0;
+    encoding[i++] = 0;
+  }
+  for (int unit in utf32CodeUnits) {
+    encoding[i++] = unit & UNICODE_BYTE_ZERO_MASK;
+    encoding[i++] = (unit >> 8) & UNICODE_BYTE_ZERO_MASK;
+    encoding[i++] = (unit >> 16) & UNICODE_BYTE_ZERO_MASK;
+    encoding[i++] = (unit >> 24) & UNICODE_BYTE_ZERO_MASK;
+  }
+  return encoding;
+}
+
+/**
+ * Joins groups of 4 bytes (0-255) UTF32-BE to produce single code points.
+ */
+List<int> _utf32beToCodePoints(List<int> utf32beEncodedBytes,
+    [int start = 0, int length = null]) {
+  if (!(start >= 0)) {
+    throw new IllegalArgumentException("start");
+  }
+
+  if (!(length == null || length >= 0)) {
+    throw new IllegalArgumentException("length");
+  }
+
+  int end = length != null ?
+      Math.min(utf32beEncodedBytes.length, start + length) :
+      utf32beEncodedBytes.length;
+
+  int i = start;
+  int lastIndex = end - 3;
+
+  if ((start < lastIndex) && utf32beEncodedBytes[start] == 0 &&
+      utf32beEncodedBytes[start + 1] == 0 &&
+      utf32beEncodedBytes[start + 2] == UNICODE_UTF_BOM_HI &&
+      utf32beEncodedBytes[start + 3] == UNICODE_UTF_BOM_LO) {
+    i += 4;
+  }
+
+  List<int> codepoints = new List<int>(((end - i)/4).ceil().toInt());
+  int j = 0;
+  while (i < lastIndex) {
+    int value = utf32beEncodedBytes[i++];
+    value = (value << 8) + utf32beEncodedBytes[i++];
+    value = (value << 8) + utf32beEncodedBytes[i++];
+    value = (value << 8) + utf32beEncodedBytes[i++];
+    codepoints[j++] = _filterValidCodepoint(value);
+  }
+  while (j < codepoints.length) {
+    codepoints[j++] = UNICODE_REPLACEMENT_CHARACTER_CODEPOINT;
+  }
+  return codepoints;
+}
+
+/**
+ * Joins groups of 4 bytes (0-255) UTF32-LE to produce single code points.
+ */
+List<int> _utf32leToCodePoints(List<int> utf32leEncodedBytes,
+    [int start = 0, int length = null]) {
+  if (!(start >= 0)) {
+    throw new IllegalArgumentException("start");
+  }
+
+  if (!(length == null || length >= 0)) {
+    throw new IllegalArgumentException("length");
+  }
+
+  int end = length != null ?
+      Math.min(utf32leEncodedBytes.length, start + length) :
+      utf32leEncodedBytes.length;
+
+  int i = start;
+  int lastIndex = end - 3;
+
+  if ((start < lastIndex) && utf32leEncodedBytes[start] == UNICODE_UTF_BOM_LO &&
+      utf32leEncodedBytes[start + 1] == UNICODE_UTF_BOM_HI &&
+      utf32leEncodedBytes[start + 2] == 0 &&
+      utf32leEncodedBytes[start + 3] == 0) {
+    i += 4;
+  }
+
+  List<int> codepoints = new List<int>(((end - i)/4).ceil().toInt());
+  int j = 0;
+  while (i < lastIndex) {
+    int value = utf32leEncodedBytes[i+3];
+    value = (value << 8) + utf32leEncodedBytes[i+2];
+    value = (value << 8) + utf32leEncodedBytes[i+1];
+    value = (value << 8) + utf32leEncodedBytes[i];
+    i += 4;
+    codepoints[j++] = _filterValidCodepoint(value);
+  }
+  while (j < codepoints.length) {
+    codepoints[j++] = UNICODE_REPLACEMENT_CHARACTER_CODEPOINT;
+  }
+  return codepoints;
+}
+
+/**
+ * Joins groups of 4 bytes (0-255) UTF32 to produce single code points.
+ */
+List<int> _utf32ToCodePoints(List<int> utf32EncodedBytes,
+    [int start = 0, int length = null]) {
+  if (!(start >= 0)) {
+    throw new IllegalArgumentException("start");
+  }
+
+  if (!(length == null || length >= 0)) {
+    throw new IllegalArgumentException("length");
+  }
+
+  int end = length != null ?
+      Math.min(utf32EncodedBytes.length, start + length) :
+      utf32EncodedBytes.length;
+
+  if ((start + 3 < end) &&
+      utf32EncodedBytes[start] == 0 &&
+      utf32EncodedBytes[start + 1] == 0 &&
+      utf32EncodedBytes[start + 2] == UNICODE_UTF_BOM_HI &&
+      utf32EncodedBytes[start + 3] == UNICODE_UTF_BOM_LO) {
+    return _utf32beToCodePoints(utf32EncodedBytes, start + 4,
+        end - (start + 4));
+  } else if ((start + 3 < end) &&
+        utf32EncodedBytes[start] == UNICODE_UTF_BOM_LO &&
+        utf32EncodedBytes[start + 1] == UNICODE_UTF_BOM_HI &&
+        utf32EncodedBytes[start + 2] == 0 &&
+        utf32EncodedBytes[start + 3] == 0) {
+    return _utf32leToCodePoints(utf32EncodedBytes, start + 4,
+        end - (start + 4));
+  } else {
+    return _utf32beToCodePoints(utf32EncodedBytes, start, end - start);
+  }
+}
+
+int _filterValidCodepoint(int codepoint) {
+  if ((codepoint >= 0 && codepoint < UNICODE_UTF16_RESERVED_LO) ||
+      (codepoint > UNICODE_UTF16_RESERVED_HI &&
+      codepoint < UNICODE_VALID_RANGE_MAX)) {
+    return codepoint;
+  } else {
+    return UNICODE_REPLACEMENT_CHARACTER_CODEPOINT;
+  }
+}