dart:convert library.

sdk/lib/convert/utf.dart

+// Copyright (c) 2013, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+part of dart.convert;
+
+// UTF-8 constants.
+const int _ONE_BYTE_LIMIT = 0x7f;   // 7 bytes
+const int _TWO_BYTE_LIMIT = 0x7ff;  // 11 bytes
+const int _THREE_BYTE_LIMIT = 0xffff;  // 16 bytes
+const int _FOUR_BYTE_LIMIT = 0x10ffff;  // 21 bytes, truncated to Unicode max.
+
+// UTF-16 constants.
+const int _SURROGATE_MASK = 0xF800;
+const int _SURROGATE_TAG_MASK = 0xFC00;
+const int _SURROGATE_VALUE_MASK = 0x3FF;
+const int _LEAD_SURROGATE_MIN = 0xD800;
+const int _TAIL_SURROGATE_MIN = 0xDC00;
+
+bool _isSurrogate(int codeUnit) =>
+    (codeUnit & _SURROGATE_MASK) == _LEAD_SURROGATE_MIN;
+bool _isLeadSurrogate(int codeUnit) =>
+    (codeUnit & _SURROGATE_TAG_MASK) == _LEAD_SURROGATE_MIN;
+bool _isTailSurrogate(int codeUnit) =>
+    (codeUnit & _SURROGATE_TAG_MASK) == _TAIL_SURROGATE_MIN;
+int _combineSurrogatePair(int lead, int tail) =>
+    0x10000 | ((lead & _SURROGATE_VALUE_MASK) << 10)
+            | (tail & _SURROGATE_VALUE_MASK);
+
+
+class _Utf8EncoderToken { const _Utf8EncoderToken(); }
+
+class Utf8Encoder extends Converter<String, List<int>> {
+  fuse(Converter other) => other.fuseInput(this);
+  fuseInput(Converter other) => new FusedConverter(other, this);
+
+  Object get id => const _Utf8EncoderToken();
+
+  ChunkedConversionSink startChunkedConversion(ChunkedConversionSink sink) {
+    return new _Utf8EncoderSink(outputInterface.adapt(sink));
+  }
+
+  ChunkedConversionInterface get inputInterface =>
+      StringConversionSink.INTERFACE;
+  ChunkedConversionInterface get outputInterface =>
+      ByteConversionSink.INTERFACE;
+}
+
+class _Utf8DecoderToken { const _Utf8DecoderToken(); }
+
+class Utf8Decoder extends Converter<List<int>, String> {
+  String convert(List<int> input) {
+    Utf8Handler handler = new Utf8Handler();
+    String result = handler.convert(input, 0, input.length);
+    if (handler.hasPartialInput) {
+      throw new ArgumentError("Incomplete UTF8 sequence");
+    }
+    return result;
+  }
+
+  fuse(Converter other) => other.fuseInput(this);
+  fuseInput(Converter other) => new FusedConverter(other, this);
+
+  Object get id => const _Utf8DecoderToken();
+
+  ChunkedConversionSink startChunkedConversion(ChunkedConversionSink sink) {
+    return new _Utf8DecoderSink(outputInterface.adapt(sink));
+  }
+
+  ChunkedConversionInterface get inputInterface =>
+      ByteConversionSink.INTERFACE;
+  ChunkedConversionInterface get outputInterface =>
+      StringConversionSink.INTERFACE;
+
+}
+
+class _Utf8DecoderSink extends ByteConversionSinkBase {
+  StringConversionSink _sink;
+  _Utf8DecoderSink(this._sink);
+
+  void addChunk(List<int> input, int start, int end, bool isLast) {
+    _sink.addUtf8(input, start, end, isLast);
+  }
+
+  void add(List<int> input) {
+    _sink.addUtf8(input, 0, input.length, false);
+  }
+
+  void close() => _sink.close();
+}
+
+class _Utf8EncoderSink extends StringConversionSinkBase {
+  ByteConversionSink _sink;
+  _Utf8EncoderSink(this._sink);
+
+  void add(String str) => addString(str, false);
+
+  void addString(String str, bool isLast) {
+    // TODO(floitsch): Use an iterator that works directly with codeUnits.
+    _sink.iterateBytes(new _Utf8EncoderIterator(str.runes.iterator), isLast);
+  }
+
+  void addRunes(List<int> runes, int start, int end, bool isLast) {
+    Iterator iterator;
+    if (start == 0 && end == runes.length) {
+      iterator = runes.iterator;
+    } else {
+      iterator = runes.getRange(start, end).iterator;
+    }
+    _sink.iterateBytes(new _Utf8EncoderIterator(iterator), isLast);
+  }
+
+  void addUtf8(List<int> utf8Units, int start, int end, bool isLast) {
+    _sink.addChunk(utf8Units, start, end, isLast);
+  }
+
+  void addAscii(List<int> codeUnits, int start, int end, bool isLast) {
+    _sink.addChunk(codeUnits, start, end, isLast);
+  }
+
+  void iterateCodeUnits(StringIterator iterator, bool isLast) {
+    _sink.iterateBytes(new _StringIteratorWrapper(iterator.asUtf8()), isLast);
+  }
+
+  void iterateRunes(StringIterator iterator, bool isLast) {
+    _sink.iterateBytes(new _StringIteratorWrapper(iterator.asUtf8()), isLast);
+  }
+
+  void iterateUtf8(StringIterator iterator, bool isLast) {
+    _sink.iterateBytes(new _StringIteratorWrapper(iterator), isLast);
+  }
+
+  void iterateIsoLatin1(StringIterator iterator, bool isLast) {
+    _sink.iterateBytes(new _StringIteratorWrapper(iterator), isLast);
+  }
+
+  void iterateAscii(StringIterator iterator, bool isLast) {
+    _sink.iterateBytes(new _StringIteratorWrapper(iterator), isLast);
+  }
+}
+
+class _StringIteratorWrapper implements ByteIterator {
+  StringIterator _iterator;
+
+  _StringIteratorWrapper(StringIterator this._iterator);
+
+  int get current => _iterator.current;
+  bool moveNext() => _iterator.moveNext();
+  int fill(List<int> list, int startIndex, int endIndex) =>
+      _iterator.fill(list, startIndex, endIndex);
+
+  List<int> convertToByteList() {
+    List<int> list = new Uint8List(32);
+    int startIndex = 0;
+    do {
+      int filled = fill(list, startIndex, list.length);
+      if (filled < list.length) {
+        List<int> result = new Uint8List(filled);
+        result.setRange(0, filled, list);
+        return result;
+      }
+      // Double in size.
+      List<int> biggerList = new Uint8List(list.length * 2);
+      biggerList.setRange(0, list.length, list);
+      startIndex = list.length;
+      list = biggerList;
+    } while(true);
+  }
+}
+
+class _Utf8EncoderIterator implements ByteIterator {
+  List<int> _carry;
+  final Iterator _runeIterator;
+  int _carryPos = 0;
+  int _current;
+
+  _Utf8EncoderIterator(this._runeIterator);
+
+  void _addRune(int rune) {
+    assert(rune > _ONE_BYTE_LIMIT);
+    if (_carry == null) {
+      _carry = new List<int>(3);
+    }
+    _carryPos = 0;
+    if (rune <= _TWO_BYTE_LIMIT) {
+      _carry[_carryPos++] = 0x80 | (rune & 0x3f);
+      _current = 0xC0 | (rune >> 6);
+    } else if (rune <= _THREE_BYTE_LIMIT) {
+      _carry[_carryPos++] = 0x80 | (rune & 0x3f);
+      _carry[_carryPos++] = 0x80 | ((rune >> 6) & 0x3f);
+      _current = 0xE0 | (rune >> 12);
+    } else {
+      assert(rune <= _FOUR_BYTE_LIMIT);
+      _carry[_carryPos++] = 0x80 | (rune & 0x3f);
+      _carry[_carryPos++] = 0x80 | ((rune >> 6) & 0x3f);
+      _carry[_carryPos++] = 0x80 | ((rune >> 12) & 0x3f);
+      _current = 0xF0 | (rune >> 18);
+    }
+  }
+
+  bool moveNext() {
+    if (_carryPos > 0) {
+      _carryPos--;
+      _current = _carry[_carryPos];
+      return true;
+    }
+    bool hasNext = _runeIterator.moveNext();
+    if (!hasNext) {
+      _current = null;
+      return false;
+    }
+    int rune = _runeIterator.current;
+    if (rune < _ONE_BYTE_LIMIT) {
+      _current = rune;
+    } else {
+      _addRune(rune);
+    }
+    return true;
+  }
+
+  int get current => _current;
+
+  int fill(List<int> list, int startIndex, int endIndex) {
+    int i = startIndex;
+    while (i < endIndex && _carryPos > 0) {
+      _carryPos--;
+      list[i++] = _carry[_carryPos];
+    }
+    while (i < endIndex - 4 && _runeIterator.moveNext()) {
+      int rune = _runeIterator.current;
+      if (rune < _ONE_BYTE_LIMIT) {
+        list[i++] = rune;
+      } else if (rune <= _TWO_BYTE_LIMIT) {
+        list[i++] = 0xC0 | (rune >> 6);
+        list[i++] = 0x80 | (rune & 0x3f);
+      } else if (rune <= _THREE_BYTE_LIMIT) {
+        list[i++] = 0xE0 | (rune >> 12);
+        list[i++] = 0x80 | ((rune >> 6) & 0x3f);
+        list[i++] = 0x80 | (rune & 0x3f);
+      } else {
+        assert(rune <= _FOUR_BYTE_LIMIT);
+        list[i++] = 0xF0 | (rune >> 18);
+        list[i++] = 0x80 | ((rune >> 12) & 0x3f);
+        list[i++] = 0x80 | ((rune >> 6) & 0x3f);
+        list[i++] = 0x80 | (rune & 0x3f);
+      }
+    }
+    while (i < endIndex && _runeIterator.moveNext()) {
+      int rune = _runeIterator.current;
+      if (rune < _ONE_BYTE_LIMIT) {
+        list[i++] = rune;
+        continue;
+      }
+      if (rune <= _TWO_BYTE_LIMIT) {
+        if (i + 2 <= endIndex) {
+          list[i++] = 0xC0 | (rune >> 6);
+          list[i++] = 0x80 | (rune & 0x3f);
+          continue;
+        }
+      } else if (rune <= _THREE_BYTE_LIMIT) {
+        if (i + 3 <= endIndex) {
+          list[i++] = 0xE0 | (rune >> 12);
+          list[i++] = 0x80 | ((rune >> 6) & 0x3f);
+          list[i++] = 0x80 | (rune & 0x3f);
+          continue;
+        }
+      } else {
+        assert(rune <= _FOUR_BYTE_LIMIT);
+        if (i + 4 <= endIndex) {
+          list[i++] = 0xF0 | (rune >> 18);
+          list[i++] = 0x80 | ((rune >> 12) & 0x3f);
+          list[i++] = 0x80 | ((rune >> 6) & 0x3f);
+          list[i++] = 0x80 | (rune & 0x3f);
+          continue;
+        }
+      }
+      // We don't have enough space to fill the list.
+      // Store the remainder in the carry.
+      _addRune(rune);
+      while (i < endIndex) {
+        _carryPos--;
+        list[i++] = _carry[_carryPos];
+      }
+    }
+    return i;
+  }
+
+  List<int> convertToByteList() {
+    List<int> list = new Uint8List(32);
+    int startIndex = 0;
+    do {
+      int filled = fill(list, startIndex, list.length);
+      if (filled < list.length) {
+        List<int> result = new Uint8List(filled);
+        result.setRange(0, filled, list);
+        return result;
+      }
+      // Double in size.
+      List<int> biggerList = new Uint8List(list.length * 2);
+      biggerList.setRange(0, list.length, list);
+      startIndex = list.length;
+      list = biggerList;
+    } while(true);
+  }
+}