String encoding utility methods and tests for Unicode, UTF-8, -16 and -32.

utils/string_encoding/Utf16.dart

Index: utils/string_encoding/Utf16.dart
diff --git a/utils/string_encoding/Utf16.dart b/utils/string_encoding/Utf16.dart
new file mode 100644
index 0000000000000000000000000000000000000000..928306bcd89c1eccb2738e2fdac4960b05a02449
--- /dev/null
+++ b/utils/string_encoding/Utf16.dart
@@ -0,0 +1,147 @@
+// Copyright (c) 2011, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+#library("UTF16");
+#import("UnicodeCore.dart");
+#import("Unicode.dart");
+
+/**
+ * Produce a String from a sequence of UTF16 encoded bytes.
+ */
+String decodeFromUtf16(List<int> bytes) {
+  List<int> codeUnits = _utf16ToUtf16CodeUnits(bytes);
+  if (is16BitCodeUnit()) {
+    return new String.fromCharCodes(codeUnits);

[Dan Rice, 2012/01/31 15:59:55]

Comment that this account for platform-specific String behavior

[dcarlson, 2012/01/31 22:11:38]

Done.

+  } else {
+    return new String.fromCharCodes(utf16CodeUnitsToCodepoints(codeUnits));
+  }
+}
+
+/**
+ * Produce a String from a sequence of UTF16-BE encoded bytes.
+ */
+String decodeFromUtf16be(List<int> bytes) {
+  List<int> codeUnits = _utf16beToUtf16CodeUnits(bytes);
+  if (is16BitCodeUnit()) {
+    return new String.fromCharCodes(codeUnits);

[Dan Rice, 2012/01/31 15:59:55]

Ditto

[dcarlson, 2012/01/31 22:11:38]

Done.

+  } else {
+    return new String.fromCharCodes(utf16CodeUnitsToCodepoints(codeUnits));
+  }
+}
+
+/**
+ * Produce a String from a sequence of UTF16-LE encoded bytes.
+ */
+String decodeFromUtf16le(List<int> bytes) {
+  List<int> codeUnits = _utf16leToUtf16CodeUnits(bytes);
+  if (is16BitCodeUnit()) {
+    return new String.fromCharCodes(codeUnits);
+  } else {
+    return new String.fromCharCodes(utf16CodeUnitsToCodepoints(codeUnits));
+  }
+}
+
+/**
+ * Produce a sequence of UTF16 encoded bytes.
+ */
+List<int> encodeAsUtf16(String str) => encodeAsUtf16be(str);
+
+/**
+ * Produce a sequence of UTF16-BE encoded bytes.
+ */
+List<int> encodeAsUtf16be(String str) {
+  List<int> utf16CodeUnits = _stringToUtf16CodeUnits(str);
+  List<int> encoding = <int>[0xfe, 0xff];

[jat, 2012/01/31 15:19:22]

Is this supposed to have BOM?  I thought UTF16-BE and UTF16-LE never included
BOM since you know the byte order by external agreement?

[Dan Rice, 2012/01/31 15:59:55]

Done.

[dcarlson, 2012/01/31 22:11:38]

It is the BOM --  not required, but currently stripping if provided (e.g. it has
a BOM).

[jat, 2012/01/31 22:46:12]

My recollection is some libraries treat the presence of BOM in the case of
UTF16-LE/BE as an error, since you are supposed to already know what it is --
otherwise, there is no value of having UTF16-LE/BE, just use UTF16 if you are
going to have BOM anyway.

I am assuming this isn't an issue for what you are interoperating with, so if
you want to leave it the way it is I am ok with that.

[dcarlson, 2012/02/01 22:18:46]

So, I'm switching things up a little. I want to provide flexibility to the user,
because I don't necessarily know about the context (e.g. the protocol). I don't
throw an exception, but include a parameter that can specify whether to strip
the BOM. 

+  for (int unit in utf16CodeUnits) {
+    encoding.add((unit & 0xff00) >> 8);
+    encoding.add(unit & 0xff);
+  }
+  return encoding;
+}
+
+/**
+ * Produce a sequence of UTF16-LE encoded bytes.
+ */
+List<int> encodeAsUtf16le(String str) {
+  List<int> utf16CodeUnits = _stringToUtf16CodeUnits(str);
+  List<int> encoding = <int>[0xff, 0xfe];
+  for (int unit in utf16CodeUnits) {
+    encoding.add(unit & 0xff);
+    encoding.add((unit & 0xff00) >> 8);
+  }
+  return encoding;
+}
+
+List<int> _stringToUtf16CodeUnits(String str) {
+  List<int> codepoints = <int>[];
+  if (is16BitCodeUnit()) {
+    return str.charCodes();
+  } else {
+    return codepointsToUtf16CodeUnits(str.charCodes());
+  }
+}
+
+/**
+ * Convert UTF-16BE encoded bytes to utf16 code units by grouping 1-2 bytes
+ * to produce the code unit (0-(2^16)-1).
+ */
+List<int> _utf16beToUtf16CodeUnits(List<int> utf16beEncodedBytes,
+    [int start = 0, int length = - 1]) {

[Dan Rice, 2012/01/31 15:59:55]

length = null

[dcarlson, 2012/01/31 22:11:38]

Done.

+  List<int> codeUnits = <int>[];
+  int end = length >= 0 ?
+      Math.min(utf16beEncodedBytes.length, start + length) :
+      utf16beEncodedBytes.length;
+  int i = start;
+  int lastIndex = end - 1;
+  while (i < lastIndex) {
+    int hi = utf16beEncodedBytes[i++];
+    int lo = utf16beEncodedBytes[i++];
+    codeUnits.add((hi * 256) + lo);

[Dan Rice, 2012/01/31 15:59:55]

use shift?

[dcarlson, 2012/01/31 22:11:38]

Done.

+  }
+  return codeUnits;
+}
+
+/**
+ * Convert UTF-16LE encoded bytes to utf16 code units by grouping 1-2 bytes
+ * to produce the code unit (0-(2^16)-1).
+ */
+List<int> _utf16leToUtf16CodeUnits(List<int> utf16leEncodedBytes,
+    [int start = 0, int length = - 1]) {

[Dan Rice, 2012/01/31 15:59:55]

length = null

[dcarlson, 2012/01/31 22:11:38]

Done.

+  List<int> codeUnits = <int>[];
+  int end = length >= 0 ?
+      Math.min(utf16leEncodedBytes.length, start + length) :
+      utf16leEncodedBytes.length;
+  int i = start;
+  int lastIndex = end - 1;
+  while (i < lastIndex) {
+    int lo = utf16leEncodedBytes[i++];
+    int hi = utf16leEncodedBytes[i++];
+    codeUnits.add((hi * 256) + lo);
+  }
+  return codeUnits;
+}
+
+/**
+ * Convert UTF-16 encoded bytes to utf16 code units by grouping 1-2 bytes
+ * to produce the code unit (0-(2^16)-1). Relies on BOM to determine
+ * endian-ness, and defaults to BE.
+ */
+List<int> _utf16ToUtf16CodeUnits(List<int> utf16EncodedBytes,
+     [int start = 0, int length = - 1]) {
+   int end = length >= 0 ?
+       Math.min(utf16EncodedBytes.length, start + length) :
+       utf16EncodedBytes.length;
+
+  if((start + 1 < end) && utf16EncodedBytes[start] == 0xfe &&

[Dan Rice, 2012/01/31 15:59:55]

use 'start + 1 < end' or 'start + 2 <= end' consistently

[dcarlson, 2012/01/31 22:11:38]

Done.

+      utf16EncodedBytes[start + 1] == 0xff) {
+    return _utf16beToUtf16CodeUnits(utf16EncodedBytes, start + 2,
+        end - (start + 2));
+  } else if((start + 2 <= end) && utf16EncodedBytes[start] == 0xff &&
+      utf16EncodedBytes[start + 1] == 0xfe) {
+    return _utf16leToUtf16CodeUnits(utf16EncodedBytes, start + 2,
+        end - (start + 2));
+  } else {
+    return _utf16beToUtf16CodeUnits(utf16EncodedBytes, start, end - start);
+  }
+}