restructure string decoding to support iterable use and include benchmarks for UTF-8 decoding.

utils/tests/string_encoding/utf8_tests.dart

 #!/usr/bin/env dart
 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 #library("utf8_tests");
 #import("dunit.dart");
 #import("../../string_encoding/unicode.dart");
 #import("../../string_encoding/unicode_core.dart");
 #source("../../string_encoding/utf8_impl.dart");
 
 void main() {
   TestSuite suite = new TestSuite();
   suite.registerTestClass(new Utf8Tests());
   suite.run();
 }
 
 class Utf8Tests extends TestClass {
   static final String testEnglishPhrase =
       "The quick brown fox jumps over the lazy dog.";
 
   static final List<int> testEnglishUtf8 = const<int> [
       0x54, 0x68, 0x65, 0x20, 0x71, 0x75, 0x69, 0x63,
       0x6b, 0x20, 0x62, 0x72, 0x6f, 0x77, 0x6e, 0x20,
       0x66, 0x6f, 0x78, 0x20, 0x6a, 0x75, 0x6d, 0x70,
       0x73, 0x20, 0x6f, 0x76, 0x65, 0x72, 0x20, 0x74,
       0x68, 0x65, 0x20, 0x6c, 0x61, 0x7a, 0x79, 0x20,
       0x64, 0x6f, 0x67, 0x2e];
-      
+
   static final String testDanishPhrase = "Quizdeltagerne spiste jordbær med " +
       "fløde mens cirkusklovnen Wolther spillede på xylofon.";
 
   static final List<int> testDanishUtf8 = const<int>[
       0x51, 0x75, 0x69, 0x7a, 0x64, 0x65, 0x6c, 0x74,
       0x61, 0x67, 0x65, 0x72, 0x6e, 0x65, 0x20, 0x73,
       0x70, 0x69, 0x73, 0x74, 0x65, 0x20, 0x6a, 0x6f,
       0x72, 0x64, 0x62, 0xc3, 0xa6, 0x72, 0x20, 0x6d,
       0x65, 0x64, 0x20, 0x66, 0x6c, 0xc3, 0xb8, 0x64,
       0x65, 0x20, 0x6d, 0x65, 0x6e, 0x73, 0x20, 0x63,
@@ skipping 77 matching lines @@
       0xe3, 0x83, 0xa4, 0xe3, 0x83, 0x9e, 0x20, 0xe3,
       0x82, 0xb1, 0xe3, 0x83, 0x95, 0xe3, 0x82, 0xb3,
       0xe3, 0x82, 0xa8, 0xe3, 0x83, 0x86, 0x20, 0xe3,
       0x82, 0xa2, 0xe3, 0x82, 0xb5, 0xe3, 0x82, 0xad,
       0xe3, 0x83, 0xa6, 0xe3, 0x83, 0xa1, 0xe3, 0x83,
       0x9f, 0xe3, 0x82, 0xb7, 0x20, 0xe3, 0x83, 0xb1,
       0xe3, 0x83, 0x92, 0xe3, 0x83, 0xa2, 0xe3, 0x82,
       0xbb, 0xe3, 0x82, 0xb9, 0xe3, 0x83, 0xb3];
 
   void registerTests(TestSuite suite) {
-    register("testUtf8bytesToCodepoints", testUtf8bytesToCodepoints, suite);
-    register("testUtf8BytesToString", testUtf8BytesToString, suite);
-    register("testEncodeToUtf8", testEncodeToUtf8, suite);
+    register("Utf8Tests.testUtf8bytesToCodepoints", testUtf8bytesToCodepoints,
+        suite);
+    register("Utf8Tests.testUtf8BytesToString", testUtf8BytesToString, suite);
+    register("Utf8Tests.testEncodeToUtf8", testEncodeToUtf8, suite);
+    register("Utf8Tests.testIterableMethods", testIterableMethods, suite);
   }
 
   void testEncodeToUtf8() {
-    Expect.listEquals(testEnglishUtf8, encodeAsUtf8(testEnglishPhrase),
+    Expect.listEquals(testEnglishUtf8, encodeUtf8(testEnglishPhrase),
         "english to utf8");
 
-    Expect.listEquals(testDanishUtf8, encodeAsUtf8(testDanishPhrase),
+    Expect.listEquals(testDanishUtf8, encodeUtf8(testDanishPhrase),
         "encode danish to utf8");
 
-    Expect.listEquals(testHebrewUtf8, encodeAsUtf8(testHebrewPhrase),
+    Expect.listEquals(testHebrewUtf8, encodeUtf8(testHebrewPhrase),
         "Hebrew to utf8");
 
-    Expect.listEquals(testRussianUtf8, encodeAsUtf8(testRussianPhrase),
+    Expect.listEquals(testRussianUtf8, encodeUtf8(testRussianPhrase),
         "Russian to utf8");
 
-    Expect.listEquals(testGreekUtf8, encodeAsUtf8(testGreekPhrase),
+    Expect.listEquals(testGreekUtf8, encodeUtf8(testGreekPhrase),
         "Greek to utf8");
 
-    Expect.listEquals(testKatakanaUtf8, encodeAsUtf8(testKatakanaPhrase),
+    Expect.listEquals(testKatakanaUtf8, encodeUtf8(testKatakanaPhrase),
         "Katakana to utf8");
   }
 
   void testUtf8bytesToCodepoints() {
     Expect.listEquals([954, 972, 963, 956, 949],
         _utf8ToCodepoints([0xce, 0xba, 0xcf, 0x8c, 0xcf,
         0x83, 0xce, 0xbc, 0xce, 0xb5]), "κόσμε");
 
     // boundary conditions: First possible sequence of a certain length
+    Expect.listEquals([], _utf8ToCodepoints([]), "no input");
     Expect.listEquals([0x0], _utf8ToCodepoints([0x0]), "0");
     Expect.listEquals([0x80], _utf8ToCodepoints([0xc2, 0x80]), "80");
     Expect.listEquals([0x800],
         _utf8ToCodepoints([0xe0, 0xa0, 0x80]), "800");
     Expect.listEquals([0x10000],
         _utf8ToCodepoints([0xf0, 0x90, 0x80, 0x80]), "10000");
     Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
         _utf8ToCodepoints([0xf8, 0x88, 0x80, 0x80, 0x80]), "200000");
     Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
         _utf8ToCodepoints([0xfc, 0x84, 0x80, 0x80, 0x80, 0x80]),
@@ skipping 262 matching lines @@
 
     // Other illegal code positions (???)
     Expect.listEquals([0xfffe], _utf8ToCodepoints([0xef, 0xbf, 0xbe]),
         "U+FFFE");
     Expect.listEquals([0xffff], _utf8ToCodepoints([0xef, 0xbf, 0xbf]),
         "U+FFFF");
   }
 
   void testUtf8BytesToString() {
     Expect.stringEquals(testEnglishPhrase,
-        decodeFromUtf8(testEnglishUtf8), "English");
+        decodeUtf8(testEnglishUtf8), "English");
 
     Expect.stringEquals(testDanishPhrase,
-        decodeFromUtf8(testDanishUtf8), "Danish");
+        decodeUtf8(testDanishUtf8), "Danish");
 
     Expect.stringEquals(testHebrewPhrase,
-        decodeFromUtf8(testHebrewUtf8), "Hebrew");
+        decodeUtf8(testHebrewUtf8), "Hebrew");
 
     Expect.stringEquals(testRussianPhrase,
-        decodeFromUtf8(testRussianUtf8), "Russian");
+        decodeUtf8(testRussianUtf8), "Russian");
 
     Expect.stringEquals(testGreekPhrase,
-        decodeFromUtf8(testGreekUtf8), "Greek");
+        decodeUtf8(testGreekUtf8), "Greek");
 
     Expect.stringEquals(testKatakanaPhrase,
-        decodeFromUtf8(testKatakanaUtf8), "Katakana");
+        decodeUtf8(testKatakanaUtf8), "Katakana");
+  }
+
+  void testIterableMethods() {
+    IterableUtf8Decoder englishDecoder = decodeUtf8AsIterable(testEnglishUtf8);
+    // get the first character
+    Expect.equals(testEnglishUtf8[0], englishDecoder.iterator().next());
+    // get the whole translation using the Iterable interface
+    Expect.stringEquals(testEnglishPhrase,
+        new String.fromCharCodes(new List<int>.from(englishDecoder)));
+
+    IterableUtf8Decoder kataDecoder = decodeUtf8AsIterable(testKatakanaUtf8);
+    // get the first character
+    Expect.equals(testKatakanaPhrase.charCodes()[0],
+        kataDecoder.iterator().next());
+    // get the whole translation using the Iterable interface
+    Expect.stringEquals(testKatakanaPhrase,
+        new String.fromCharCodes(new List<int>.from(kataDecoder)));
   }
 }