Fix UTF8 encoder for Unicode runes > 0xFFFF.

sdk/lib/convert/utf.dart

 // Copyright (c) 2013, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 part of dart.convert;
 
 /** The Unicode Replacement character `U+FFFD` (�). */
 const UNICODE_REPLACEMENT_CHARACTER_RUNE = 0xFFFD;
 
+/** The Unicode Byte Order Marker (BOM) character `U+FEFF`. */
+const UNICODE_BOM_CHARACTER_RUNE = 0xFEFF;
+
 /**
  * An instance of the default implementation of the [Utf8Codec].
  *
  * This instance provides a convenient access to the most common UTF-8
  * use cases.
  *
  * Examples:
  *
  *     var encoded = UTF8.encode("Îñţérñåţîöñåļîžåţîờñ");
  *     var decoded = UTF8.decode([0x62, 0x6c, 0xc3, 0xa5, 0x62, 0xc3, 0xa6,
@@ skipping 21 matching lines @@
    */
   const Utf8Codec({ bool allowMalformed: false })
       : _allowMalformed = allowMalformed;
 
   String get name => "utf-8";
 
   /**
    * Decodes the UTF-8 [codeUnits] (a list of unsigned 8-bit integers) to the
    * corresponding string.
    *
+   * If the [codeUnits] start with a leading [UNICODE_BOM_CHARACTER_RUNE] this
+   * character is discarded.
+   *
    * If [allowMalformed] is `true` the decoder replaces invalid (or
    * unterminated) character sequences with the Unicode Replacement character
    * `U+FFFD` (�). Otherwise it throws a [FormatException].
    *
    * If [allowMalformed] is not given, it defaults to the `allowMalformed` that
    * was used to instantiate `this`.
    */
   String decode(List<int> codeUnits, { bool allowMalformed }) {
     if (allowMalformed == null) allowMalformed = _allowMalformed;
     return new Utf8Decoder(allowMalformed: allowMalformed).convert(codeUnits);
@@ skipping 235 matching lines @@
    * If it is `true` [convert] replaces invalid (or unterminated) character
    * sequences with the Unicode Replacement character `U+FFFD` (�). Otherwise
    * it throws a [FormatException].
    */
   const Utf8Decoder({ bool allowMalformed: false })
       : this._allowMalformed = allowMalformed;
 
   /**
    * Converts the UTF-8 [codeUnits] (a list of unsigned 8-bit integers) to the
    * corresponding string.
+   *
+   * If the [codeUnits] start with a leading [UNICODE_BOM_CHARACTER_RUNE] this
+   * character is discarded.
    */
   String convert(List<int> codeUnits) {
     StringBuffer buffer = new StringBuffer();
     _Utf8Decoder decoder = new _Utf8Decoder(buffer, _allowMalformed);
     decoder.convert(codeUnits, 0, codeUnits.length);
     decoder.close();
     return buffer.toString();
   }
 
   /**
@@ skipping 23 matching lines @@
 const int _THREE_BYTE_LIMIT = 0xffff;  // 16 bits
 const int _FOUR_BYTE_LIMIT = 0x10ffff;  // 21 bits, truncated to Unicode max.
 
 // UTF-16 constants.
 const int _SURROGATE_MASK = 0xF800;
 const int _SURROGATE_TAG_MASK = 0xFC00;
 const int _SURROGATE_VALUE_MASK = 0x3FF;
 const int _LEAD_SURROGATE_MIN = 0xD800;
 const int _TAIL_SURROGATE_MIN = 0xDC00;
 
-const int _REPLACEMENT_CHARACTER = 0xFFFD;
-const int _BOM_CHARACTER = 0xFEFF;
-
 bool _isSurrogate(int codeUnit) =>
     (codeUnit & _SURROGATE_MASK) == _LEAD_SURROGATE_MIN;
 bool _isLeadSurrogate(int codeUnit) =>
     (codeUnit & _SURROGATE_TAG_MASK) == _LEAD_SURROGATE_MIN;
 bool _isTailSurrogate(int codeUnit) =>
     (codeUnit & _SURROGATE_TAG_MASK) == _TAIL_SURROGATE_MIN;
 int _combineSurrogatePair(int lead, int tail) =>
-    0x10000 | ((lead & _SURROGATE_VALUE_MASK) << 10)
+    0x10000 + ((lead & _SURROGATE_VALUE_MASK) << 10)
             | (tail & _SURROGATE_VALUE_MASK);
 
 
 /**
  * Decodes UTF-8.
  *
  * The decoder handles chunked input.
  */
 // TODO(floitsch): make this class public.
 class _Utf8Decoder {
@@ skipping 23 matching lines @@
    * Flushes this decoder as if closed.
    *
    * This method throws if the input was partial and the decoder was
    * constructed with `allowMalformed` set to `false`.
    */
   void flush() {
     if (hasPartialInput) {
       if (!_allowMalformed) {
         throw new FormatException("Unfinished UTF-8 octet sequence");
       }
-      _stringSink.writeCharCode(_REPLACEMENT_CHARACTER);
+      _stringSink.writeCharCode(UNICODE_REPLACEMENT_CHARACTER_RUNE);
       _value = 0;
       _expectedUnits = 0;
       _extraUnits = 0;
     }
   }
 
   void convert(List<int> codeUnits, int startIndex, int endIndex) {
     int value = _value;
     int expectedUnits = _expectedUnits;
     int extraUnits = _extraUnits;
     _value = 0;
     _expectedUnits = 0;
     _extraUnits = 0;
 
     int i = startIndex;
     loop: while (true) {
       multibyte: if (expectedUnits > 0) {
         do {
           if (i == endIndex) {
             break loop;
           }
           int unit = codeUnits[i];
           if ((unit & 0xC0) != 0x80) {
             expectedUnits = 0;
             if (!_allowMalformed) {
               throw new FormatException(
                   "Bad UTF-8 encoding 0x${unit.toRadixString(16)}");
             }
             _isFirstCharacter = false;
-            _stringSink.writeCharCode(_REPLACEMENT_CHARACTER);
+            _stringSink.writeCharCode(UNICODE_REPLACEMENT_CHARACTER_RUNE);
             break multibyte;
           } else {
             value = (value << 6) | (unit & 0x3f);
             expectedUnits--;
             i++;
           }
         } while (expectedUnits > 0);
         if (value <= _LIMITS[extraUnits - 1]) {
           // Overly long encoding. The value could be encoded with a shorter
           // encoding.
           if (!_allowMalformed) {
             throw new FormatException(
                 "Overlong encoding of 0x${value.toRadixString(16)}");
           }
           expectedUnits = extraUnits = 0;
-          value = _REPLACEMENT_CHARACTER;
+          value = UNICODE_REPLACEMENT_CHARACTER_RUNE;
         }
         if (value > _FOUR_BYTE_LIMIT) {
           if (!_allowMalformed) {
             throw new FormatException("Character outside valid Unicode range: "
                                       "0x${value.toRadixString(16)}");
           }
-          value = _REPLACEMENT_CHARACTER;
+          value = UNICODE_REPLACEMENT_CHARACTER_RUNE;
         }
-        if (!_isFirstCharacter || value != _BOM_CHARACTER) {
+        if (!_isFirstCharacter || value != UNICODE_BOM_CHARACTER_RUNE) {
           _stringSink.writeCharCode(value);
         }
         _isFirstCharacter = false;
       }
 
       while (i < endIndex) {
         int unit = codeUnits[i++];
         // TODO(floitsch): the way we test we could potentially allow
         // units that are too large, if they happen to have the
         // right bit-pattern. (Same is true for the multibyte loop above).
         // TODO(floitsch): optimize this loop. See:
         // https://codereview.chromium.org/22929022/diff/1/sdk/lib/convert/utf.dart?column_width=80
         if (unit < 0) {
           // TODO(floitsch): should this be unit <= 0 ?
           if (!_allowMalformed) {
             throw new FormatException(
                 "Negative UTF-8 code unit: -0x${(-unit).toRadixString(16)}");
           }
-          _stringSink.writeCharCode(_REPLACEMENT_CHARACTER);
+          _stringSink.writeCharCode(UNICODE_REPLACEMENT_CHARACTER_RUNE);
         } else if (unit <= _ONE_BYTE_LIMIT) {
           _isFirstCharacter = false;
           _stringSink.writeCharCode(unit);
         } else {
           if ((unit & 0xE0) == 0xC0) {
             value = unit & 0x1F;
             expectedUnits = extraUnits = 1;
             continue loop;
           }
           if ((unit & 0xF0) == 0xE0) {
             value = unit & 0x0F;
             expectedUnits = extraUnits = 2;
             continue loop;
           }
           // 0xF5, 0xF6 ... 0xFF never appear in valid UTF-8 sequences.
           if ((unit & 0xF8) == 0xF0 && unit < 0xF5) {
             value = unit & 0x07;
             expectedUnits = extraUnits = 3;
             continue loop;
           }
           if (!_allowMalformed) {
             throw new FormatException(
                 "Bad UTF-8 encoding 0x${unit.toRadixString(16)}");
           }
-          value = _REPLACEMENT_CHARACTER;
+          value = UNICODE_REPLACEMENT_CHARACTER_RUNE;
           expectedUnits = extraUnits = 0;
           _isFirstCharacter = false;
           _stringSink.writeCharCode(value);
         }
       }
       break loop;
     }
     if (expectedUnits > 0) {
       _value = value;
       _expectedUnits = expectedUnits;
       _extraUnits = extraUnits;
     }
   }
 }