String encoding utility methods and tests for Unicode, UTF-8, -16 and -32.

utils/string_encoding/Utf8_impl.dart

Index: utils/string_encoding/Utf8_impl.dart
diff --git a/utils/string_encoding/Utf8_impl.dart b/utils/string_encoding/Utf8_impl.dart
new file mode 100644
index 0000000000000000000000000000000000000000..014ed4ce5315ac193b8413b33e5bfff570f162bc
--- /dev/null
+++ b/utils/string_encoding/Utf8_impl.dart
@@ -0,0 +1,146 @@
+// Copyright (c) 2011, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+/**
+ * Produce a sequence of UTF8 encoded bytes from the provided string.
+ */
+List<int> encodeAsUtf8(String str) =>
+  _codepointsToUtf8(stringToCodepoints(str));
+
+/**
+ * Produce a String from a sequence of UTF8 encoded bytes.
+ */
+String decodeFromUtf8(List<int> bytes) =>
+    codepointsToString(_utf8ToCodepoints(bytes, 0, bytes.length));
+
+int _addToEncoding(int start, int bytes, int value, List<int> buffer) {
+  while(bytes > 0) {
+    buffer[start + bytes] = 0x80 + (value & 0x3f);
+    value = value >> 6;
+    bytes--;
+  }
+  return value;
+}
+
+/**
+ * Encode code points as UTF8 code units.
+ */
+List<int> _codepointsToUtf8(List<int> codepoints,
+    [int start = 0, int length = -1]) {
+
+  List<int> encoded = <int>[];
+  int end = length >= 0 ? Math.min(codepoints.length, start + length) :
+      codepoints.length;
+  int i = start;
+  while (i < end) {
+    int value = codepoints[i++];
+    int insertAt = encoded.length;
+    if (value < 0 || value > 0x7fffffff) {
+      encoded.addAll([0xef, 0xbf, 0xbd]);
+    } else if (value <= 0x7f) {
+      encoded.add(value);
+    } else if (value <= 0x7ff) {
+      encoded.insertRange(encoded.length, 2, 99);
+      encoded[insertAt] = 0xc0 |
+          (0x1f & _addToEncoding(insertAt, 1, value, encoded));
+    } else if (value <= 0xffff) {
+      encoded.insertRange(encoded.length, 3, 99);
+      encoded[insertAt] = 0xe0 |
+          (0xf & _addToEncoding(insertAt, 2, value, encoded));
+    } else if (value <= 0x1fffff) {
+      encoded.insertRange(encoded.length, 4, 0);
+      encoded[insertAt] = 0xf0 |
+          (0x7 & _addToEncoding(insertAt, 3, value, encoded));
+    } else if (value <= 0x3ffffff) {
+      encoded.insertRange(encoded.length, 5, 0);
+      encoded[insertAt] = 0xf8 |
+          (0x3 & _addToEncoding(insertAt, 4, value, encoded));
+    } else if (value <= 0x7fffffff) {
+      encoded.insertRange(encoded.length, 6, 0);
+      encoded[insertAt] = 0xfc |
+          (0x1 & _addToEncoding(insertAt, 5, value, encoded));
+    }
+  }
+  return encoded;
+}
+
+/**
+ * UTF-8 bytes are utf8 code units, so we can go directly from
+ * encoded bytes to code points.
+ */
+List<int> _utf8ToCodepoints(List<int> utf8EncodedBytes, [int start = 0,
+    int length = -1]) {
+  List<int> codepointBuffer = <int>[];
+  int end = length >= 0 ? Math.min(utf8EncodedBytes.length, start + length) :
+      utf8EncodedBytes.length;
+  int i = start;
+  while (i < end) {
+    int value = utf8EncodedBytes[i++];
+    if (value >= 0x0) {
+      if (value < 0x80) {
+        codepointBuffer.add(value);
+      } else if (value < 0xc0) {
+        codepointBuffer.add(REPLACEMENT_CHARACTER_CODEPOINT);
+        continue;
+      } else {
+        int additionalBytes = 0;
+        if (value < 0xe0) {
+          value -= 0xc0;
+          additionalBytes = 1;
+        } else if (value < 0xf0) {
+          value -= 0xe0;
+          additionalBytes = 2;
+        } else if (value < 0xf8) {
+          value -= 0xf0;
+          additionalBytes = 3;
+        } else if (value < 0xfc) {
+          value -= 0xf8;
+          additionalBytes = 4;
+        } else if (value < 0xfe) {
+          value -= 0xfc;
+          additionalBytes = 5;
+        } else {
+          codepointBuffer.add(REPLACEMENT_CHARACTER_CODEPOINT);
+          continue;
+        }

[jat, 2012/01/31 15:19:22]

Do you want to reject over-long encodings?  They can be used to subvert some
checks (ie, encoding an ASCII < as a multi-byte sequence to avoid simplistic
HTML sanitizers).

[dcarlson, 2012/01/31 22:11:38]

:) I do. See the tests for checks on overlong encodings as confirmation. The
filtering takes place below.

+        int j = 0;
+        while (j < additionalBytes && i < end) {
+          int nextValue = utf8EncodedBytes[i++];
+          if (nextValue >= 0x80 && nextValue < 0xc0) {
+            // equiv to (value << 6) | (nextValue & 0x3f)
+            value = (value * 0x40) + (nextValue - 0x80);
+          } else {
+            // if sequence-starting code unit, reposition cursor to start here
+            if (nextValue >= 0xc0) i--;
+            break;
+          }
+          j++;
+        }
+        if (j == additionalBytes &&
+            (value < 0xd800 || value > 0xdfff)) {
+          if(additionalBytes == 1 && value > 0x7f) {
+            codepointBuffer.add(value);
+          } else if(additionalBytes == 2 && value > 0x7ff) {
+            codepointBuffer.add(value);
+          } else if(additionalBytes == 3 && value > 0xffff) {
+            codepointBuffer.add(value);
+          } else if(additionalBytes == 4 && value > 0x1fffff) {
+            codepointBuffer.add(value);
+          } else if(additionalBytes == 5 && value > 0x3ffffff) {
+            codepointBuffer.add(value);
+          } else {
+            codepointBuffer.add(REPLACEMENT_CHARACTER_CODEPOINT);
+          }
+        } else {
+          codepointBuffer.add(REPLACEMENT_CHARACTER_CODEPOINT);
+          continue;
+        }
+      }
+    } else {
+      codepointBuffer.add(REPLACEMENT_CHARACTER_CODEPOINT);
+      continue;
+    }
+  }
+  return codepointBuffer;
+}