Optimize special cases in utf8 and utf16 where input can be copied to output.

utils/string_encoding/utf8_impl.dart

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 final int _UTF8_ONE_BYTE_MAX = 0x7f;
 final int _UTF8_TWO_BYTE_MAX = 0x7ff;
 final int _UTF8_THREE_BYTE_MAX = 0xffff;
 
 final int _UTF8_LO_SIX_BIT_MASK = 0x3f;
 
@@ skipping 112 matching lines @@
   }
 
   if (!(length == null || length >= 0)) {
     throw new IllegalArgumentException("length");
   }
 
   int end = length != null ?
       Math.min(utf8EncodedBytes.length, offset + length) :
       utf8EncodedBytes.length;
 
-  void addReplacementCodepoint(void f(int v), int replacementCodepoint) {
-    if(replacementCodepoint != null) {
-      f(replacementCodepoint);
-    } else {
-      throw new IllegalArgumentException("Invalid encoding");
-    }
-  }
-
-  void apply(void f(int v)) {
+  void decode(void f(int v)) {
     int i = offset;
     while (i < end) {
       int value = utf8EncodedBytes[i++];
-      if (value >= 0) {
-        if (value <= _UTF8_ONE_BYTE_MAX) {
-          f(value);
-        } else if (value < _UTF8_FIRST_BYTE_OF_TWO_BASE) {
-          addReplacementCodepoint(f, replacementCodepoint);
+      if (value < 0) {
+        f(null);
+        continue;
+      }
+
+      if (value <= _UTF8_ONE_BYTE_MAX) {
+        f(value);
+      } else if (value < _UTF8_FIRST_BYTE_OF_TWO_BASE) {
+        f(null);
+        continue;
+      } else {
+        int additionalBytes = 0;
+        if (value < _UTF8_FIRST_BYTE_OF_THREE_BASE) {
+          value -= _UTF8_FIRST_BYTE_OF_TWO_BASE;
+          additionalBytes = 1;
+        } else if (value < _UTF8_FIRST_BYTE_OF_FOUR_BASE) {
+          value -= _UTF8_FIRST_BYTE_OF_THREE_BASE;
+          additionalBytes = 2;
+        } else if (value < _UTF8_FIRST_BYTE_OF_FIVE_BASE) {
+          value -= _UTF8_FIRST_BYTE_OF_FOUR_BASE;
+          additionalBytes = 3;
+        } else if (value < _UTF8_FIRST_BYTE_OF_SIX_BASE) {
+          value -= _UTF8_FIRST_BYTE_OF_FIVE_BASE;
+          additionalBytes = 4;
+        } else if (value < _UTF8_FIRST_BYTE_BOUND_EXCL) {
+          value -= _UTF8_FIRST_BYTE_OF_SIX_BASE;
+          additionalBytes = 5;
+        } else {
+          f(null);
           continue;
+        }
+        int j = 0;
+        while (j < additionalBytes && i < end) {
+          int nextValue = utf8EncodedBytes[i++];
+          if (nextValue > _UTF8_ONE_BYTE_MAX &&
+              nextValue < _UTF8_FIRST_BYTE_OF_TWO_BASE) {
+            value = (value << 6) | (nextValue & _UTF8_LO_SIX_BIT_MASK);
+          } else {
+            // if sequence-starting code unit, reposition cursor to start here
+            if (nextValue >= _UTF8_FIRST_BYTE_OF_TWO_BASE) {
+              i--;
+            }
+            break;
+          }
+          j++;
+        }
+        if (j == additionalBytes && (
+            value < UNICODE_UTF16_RESERVED_LO ||
+            value > UNICODE_UTF16_RESERVED_HI)) {
+          if ((additionalBytes == 1 && value > _UTF8_ONE_BYTE_MAX) ||
+              (additionalBytes == 2 && value > _UTF8_TWO_BYTE_MAX) ||
+              (additionalBytes == 3 && value > _UTF8_THREE_BYTE_MAX &&
+                value <= UNICODE_VALID_RANGE_MAX)) {
+            f(value);
+          } else {
+            f(null);
+          }
         } else {
-          int additionalBytes = 0;
-          if (value < _UTF8_FIRST_BYTE_OF_THREE_BASE) {
-            value -= _UTF8_FIRST_BYTE_OF_TWO_BASE;
-            additionalBytes = 1;
-          } else if (value < _UTF8_FIRST_BYTE_OF_FOUR_BASE) {
-            value -= _UTF8_FIRST_BYTE_OF_THREE_BASE;
-            additionalBytes = 2;
-          } else if (value < _UTF8_FIRST_BYTE_OF_FIVE_BASE) {
-            value -= _UTF8_FIRST_BYTE_OF_FOUR_BASE;
-            additionalBytes = 3;
-          } else if (value < _UTF8_FIRST_BYTE_OF_SIX_BASE) {
-            value -= _UTF8_FIRST_BYTE_OF_FIVE_BASE;
-            additionalBytes = 4;
-          } else if (value < _UTF8_FIRST_BYTE_BOUND_EXCL) {
-            value -= _UTF8_FIRST_BYTE_OF_SIX_BASE;
-            additionalBytes = 5;
-          } else {
-            addReplacementCodepoint(f, replacementCodepoint);
-            continue;
-          }
-          int j = 0;
-          while (j < additionalBytes && i < end) {
-            int nextValue = utf8EncodedBytes[i++];
-            if (nextValue > _UTF8_ONE_BYTE_MAX &&
-                nextValue < _UTF8_FIRST_BYTE_OF_TWO_BASE) {
-              value = (value << 6) | (nextValue & _UTF8_LO_SIX_BIT_MASK);
-            } else {
-              // if sequence-starting code unit, reposition cursor to start here
-              if (nextValue >= _UTF8_FIRST_BYTE_OF_TWO_BASE) {
-                i--;
-              }
-              break;
-            }
-            j++;
-          }
-          if (j == additionalBytes && (
-              value < UNICODE_UTF16_RESERVED_LO ||
-              value > UNICODE_UTF16_RESERVED_HI)) {
-            if ((additionalBytes == 1 && value > _UTF8_ONE_BYTE_MAX) ||
-                (additionalBytes == 2 && value > _UTF8_TWO_BYTE_MAX) ||
-                (additionalBytes == 3 && value > _UTF8_THREE_BYTE_MAX &&
-                  value <= UNICODE_VALID_RANGE_MAX)) {
-              f(value);
-            } else {
-              addReplacementCodepoint(f, replacementCodepoint);
-            }
-          } else {
-            addReplacementCodepoint(f, replacementCodepoint);
-            continue;
-          }
+          f(null);
+          continue;
         }
-      } else {
-        addReplacementCodepoint(f, replacementCodepoint);
-        continue;
       }
     }
   }
 
   int codepointBufferLength = 0;

[Søren Gjesse, 2012/02/06 15:45:44]

Please add a comment here regarding the function passed and the ASCII
optimization.

[dcarlson, 2012/02/06 22:11:29]

Done.

-  apply(void _(int value) {
+  bool hasReplacements = false;
+  decode(void _(int value) {
       codepointBufferLength++;
+      if (value == null) {
+        hasReplacements = true;
+      }
   });
 
+  int _length = end - offset;
   List<int> codepointBuffer = new List<int>(codepointBufferLength);
-  int i = 0;
-  apply(void _(int value) {
-      codepointBuffer[i++] = value;
-  });
+  if (_length == codepointBufferLength && !hasReplacements) {
+    codepointBuffer.setRange(0, _length, utf8EncodedBytes, offset);
+  } else {
+    int i = 0;
+    decode(
+      void _(int value) {
+        if (value != null) {
+          codepointBuffer[i++] = value;
+        } else {
+          if (replacementCodepoint != null) {
+            codepointBuffer[i++] = replacementCodepoint;
+          } else {
+            throw new IllegalArgumentException("Invalid encoding");
+          }
+        }
+      }
+    );
+  }
   return codepointBuffer;
 }