String encoding utility methods and tests for Unicode, UTF-8, -16 and -32.

utils/string_encoding/Utf32.dart

Index: utils/string_encoding/Utf32.dart
diff --git a/utils/string_encoding/Utf32.dart b/utils/string_encoding/Utf32.dart
new file mode 100644
index 0000000000000000000000000000000000000000..4560a7a4d4ba67ee0ccdf7259568dd1fa6e597de
--- /dev/null
+++ b/utils/string_encoding/Utf32.dart
@@ -0,0 +1,158 @@
+// Copyright (c) 2011, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+#library("UTF32");
+#import("UnicodeCore.dart");
+#import("Unicode.dart");
+
+/**
+ * Produce a String from a sequence of UTF32 encoded bytes.
+ */
+String decodeFromUtf32(List<int> bytes) =>
+    codepointsToString(_utf32ToCodePoints(bytes));
+
+/**
+ * Produce a String from a sequence of UTF32-BE encoded bytes.
+ */
+String decodeFromUtf32be(List<int> bytes) =>
+    codepointsToString(_utf32beToCodePoints(bytes));
+
+/**
+ * Produce a String from a sequence of UTF32-LE encoded bytes.
+ */
+String decodeFromUtf32le(List<int> bytes) =>
+    codepointsToString(_utf32leToCodePoints(bytes));
+
+/**
+ * Produce a sequence of UTF32 encoded bytes.
+ */
+List<int> encodeAsUtf32(String str) => encodeAsUtf32be(str);
+
+/**
+ * Produce a sequence of UTF32-BE encoded bytes.
+ */
+List<int> encodeAsUtf32be(String str) {

[Dan Rice, 2012/01/31 15:59:55]

, [bool writeBom = false]

[dcarlson, 2012/01/31 22:11:38]

Done.

+  List<int> utf32CodeUnits = stringToCodepoints(str);
+  List<int> encoding = <int>[0, 0, 0xfe, 0xff];

[jat, 2012/01/31 15:19:22]

Same comment as UTF16-BE/LE here.

[dcarlson, 2012/01/31 22:11:38]

OK

+  for (int unit in utf32CodeUnits) {
+    encoding.add((unit >> 24) & 0xff);
+    encoding.add((unit >> 16) & 0xff);
+    encoding.add((unit >> 8) & 0xff);
+    encoding.add(unit & 0xff);
+  }
+  return encoding;
+}
+
+/**
+ * Produce a sequence of UTF32-LE encoded bytes.
+ */
+List<int> encodeAsUtf32le(String str) {
+  List<int> utf32CodeUnits = stringToCodepoints(str);
+  List<int> encoding = <int>[0xff, 0xfe, 0, 0];
+  for (int unit in utf32CodeUnits) {
+    encoding.add((unit) & 0xff);
+    encoding.add((unit >> 8) & 0xff);
+    encoding.add((unit >> 16) & 0xff);
+    encoding.add((unit >> 24) & 0xff);
+  }
+  return encoding;
+}
+
+/**
+ * Joins groups of 4 bytes (0-255) UTF32-BE to produce single code points.
+ */
+List<int> _utf32beToCodePoints(List<int> utf32beEncodedBytes,
+      [int start = 0, int length = - 1]) {
+  List<int> codepoints = <int>[];
+  int end = length >= 0 ?
+      Math.min(utf32beEncodedBytes.length, start + length) :
+      utf32beEncodedBytes.length;
+
+  int i = start;
+  int lastIndex = end - 3;
+
+  if((start + 3 < end) && utf32beEncodedBytes[start] == 0 &&
+      utf32beEncodedBytes[start + 1] == 0 &&
+      utf32beEncodedBytes[start + 2] == 0xfe &&
+      utf32beEncodedBytes[start + 3] == 0xff) {
+    i += 4;
+  }
+
+  while (i < lastIndex) {
+    int value = utf32beEncodedBytes[i++];
+    value = (256 * value) + utf32beEncodedBytes[i++];
+    value = (256 * value) + utf32beEncodedBytes[i++];
+    value = (256 * value) + utf32beEncodedBytes[i++];
+    codepoints.add(_filterValidCodepoint(value));
+  }
+  return codepoints;
+}
+
+/**
+ * Joins groups of 4 bytes (0-255) UTF32-LE to produce single code points.
+ */
+List<int> _utf32leToCodePoints(List<int> utf32leEncodedBytes,
+       [int start = 0, int length = - 1]) {
+   List<int> codepoints = <int>[];
+   int end = length >= 0 ?
+       Math.min(utf32leEncodedBytes.length, start + length) :
+       utf32leEncodedBytes.length;
+
+   int i = start;
+   int lastIndex = end - 3;
+
+   if((start + 3 < end) && utf32leEncodedBytes[start] == 0xff &&
+       utf32leEncodedBytes[start + 1] == 0xfe &&
+       utf32leEncodedBytes[start + 2] == 0 &&
+       utf32leEncodedBytes[start + 3] == 0) {
+     i += 4;
+   }
+
+   while (i < lastIndex) {
+     int value = utf32leEncodedBytes[i+3];
+     value = (256 * value) + utf32leEncodedBytes[i+2];
+     value = (256 * value) + utf32leEncodedBytes[i+1];
+     value = (256 * value) + utf32leEncodedBytes[i];
+     i += 4;
+     codepoints.add(_filterValidCodepoint(value));
+   }
+   return codepoints;
+ }
+
+/**
+ * Joins groups of 4 bytes (0-255) UTF32 to produce single code points.
+ */
+List<int> _utf32ToCodePoints(List<int> utf32EncodedBytes,
+    [int start = 0, int length = - 1]) {
+  int end = length >= 0 ?
+      Math.min(utf32EncodedBytes.length, start + length) :
+      utf32EncodedBytes.length;
+
+  if((start + 3 < end) &&
+      utf32EncodedBytes[start] == 0 &&
+      utf32EncodedBytes[start + 1] == 0 &&
+      utf32EncodedBytes[start + 2] == 0xfe &&
+      utf32EncodedBytes[start + 3] == 0xff) {
+    return _utf32beToCodePoints(utf32EncodedBytes, start + 4,
+        end - (start + 4));
+  } else if((start + 3 < end) &&
+        utf32EncodedBytes[start] == 0xff &&
+        utf32EncodedBytes[start + 1] == 0xfe &&
+        utf32EncodedBytes[start + 2] == 0 &&
+        utf32EncodedBytes[start + 3] == 0) {
+    return _utf32leToCodePoints(utf32EncodedBytes, start + 4,
+        end - (start + 4));
+  } else {
+    return _utf32beToCodePoints(utf32EncodedBytes, start, end - start);
+  }
+}
+
+int _filterValidCodepoint(int codepoint) {
+  if ((codepoint >= 0 && codepoint < 0xd800) ||
+      (codepoint >= 0x10000 && codepoint < 0x110000)) {
+    return codepoint;
+  } else {
+    return REPLACEMENT_CHARACTER_CODEPOINT;
+  }
+}