| OLD | NEW |
| (Empty) |
| 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | |
| 2 // for details. All rights reserved. Use of this source code is governed by a | |
| 3 // BSD-style license that can be found in the LICENSE file. | |
| 4 | |
| 5 final int _UTF8_ONE_BYTE_MAX = 0x7f; | |
| 6 final int _UTF8_TWO_BYTE_MAX = 0x7ff; | |
| 7 final int _UTF8_THREE_BYTE_MAX = 0xffff; | |
| 8 | |
| 9 final int _UTF8_LO_SIX_BIT_MASK = 0x3f; | |
| 10 | |
| 11 final int _UTF8_FIRST_BYTE_OF_TWO_BASE = 0xc0; | |
| 12 final int _UTF8_FIRST_BYTE_OF_THREE_BASE = 0xe0; | |
| 13 final int _UTF8_FIRST_BYTE_OF_FOUR_BASE = 0xf0; | |
| 14 final int _UTF8_FIRST_BYTE_OF_FIVE_BASE = 0xf8; | |
| 15 final int _UTF8_FIRST_BYTE_OF_SIX_BASE = 0xfc; | |
| 16 | |
| 17 final int _UTF8_FIRST_BYTE_OF_TWO_MASK = 0x1f; | |
| 18 final int _UTF8_FIRST_BYTE_OF_THREE_MASK = 0xf; | |
| 19 final int _UTF8_FIRST_BYTE_OF_FOUR_MASK = 0x7; | |
| 20 | |
| 21 final int _UTF8_FIRST_BYTE_BOUND_EXCL = 0xfe; | |
| 22 final int _UTF8_SUBSEQUENT_BYTE_BASE = 0x80; | |
| 23 | |
| 24 /** | |
| 25 * Produce a String from a sequence of UTF-8 encoded bytes. The parameters | |
| 26 * allow an offset into a list of bytes (as int), limiting the length of the | |
| 27 * values be decoded and the ability of override the default Unicode | |
| 28 * replacement character. Set the replacementCharacter to null to throw an | |
| 29 * IllegalArgumentException rather than replace the bad value. | |
| 30 */ | |
| 31 String decodeFromUtf8(List<int> bytes, [int offset = 0, int length, | |
| 32 int replacementCodepoint = UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]) => | |
| 33 codepointsToString(_utf8ToCodepoints( | |
| 34 bytes, offset, length, replacementCodepoint)); | |
| 35 | |
| 36 /** | |
| 37 * Produce a sequence of UTF-8 encoded bytes from the provided string. | |
| 38 */ | |
| 39 List<int> encodeAsUtf8(String str) => | |
| 40 _codepointsToUtf8(stringToCodepoints(str)); | |
| 41 | |
| 42 int _addToEncoding(int offset, int bytes, int value, List<int> buffer) { | |
| 43 while(bytes > 0) { | |
| 44 buffer[offset + bytes] = _UTF8_SUBSEQUENT_BYTE_BASE | | |
| 45 (value & _UTF8_LO_SIX_BIT_MASK); | |
| 46 value = value >> 6; | |
| 47 bytes--; | |
| 48 } | |
| 49 return value; | |
| 50 } | |
| 51 | |
| 52 /** | |
| 53 * Encode code points as UTF-8 code units. | |
| 54 */ | |
| 55 List<int> _codepointsToUtf8( | |
| 56 List<int> codepoints, [int offset = 0, int length]) { | |
| 57 if (!(offset >= 0)) { | |
| 58 throw new IllegalArgumentException("offset"); | |
| 59 } | |
| 60 | |
| 61 if (!(length == null || length >= 0)) { | |
| 62 throw new IllegalArgumentException("length"); | |
| 63 } | |
| 64 | |
| 65 int end = length != null ? | |
| 66 Math.min(codepoints.length, offset + length) : | |
| 67 codepoints.length; | |
| 68 | |
| 69 int encodedLength = 0; | |
| 70 for (int i = offset; i < end; i++) { | |
| 71 int value = codepoints[i]; | |
| 72 if (value < 0 || value > UNICODE_VALID_RANGE_MAX) { | |
| 73 encodedLength += 3; | |
| 74 } else if (value <= _UTF8_ONE_BYTE_MAX) { | |
| 75 encodedLength++; | |
| 76 } else if (value <= _UTF8_TWO_BYTE_MAX) { | |
| 77 encodedLength += 2; | |
| 78 } else if (value <= _UTF8_THREE_BYTE_MAX) { | |
| 79 encodedLength += 3; | |
| 80 } else if (value <= UNICODE_VALID_RANGE_MAX) { | |
| 81 encodedLength += 4; | |
| 82 } | |
| 83 } | |
| 84 | |
| 85 List<int> encoded = new List<int>(encodedLength); | |
| 86 int insertAt = 0; | |
| 87 for (int i = offset; i < end; i++) { | |
| 88 int value = codepoints[i]; | |
| 89 if (value < 0 || value > UNICODE_VALID_RANGE_MAX) { | |
| 90 encoded.setRange(insertAt, 3, [0xef, 0xbf, 0xbd]); | |
| 91 insertAt += 3; | |
| 92 } else if (value <= _UTF8_ONE_BYTE_MAX) { | |
| 93 encoded[insertAt] = value; | |
| 94 insertAt++; | |
| 95 } else if (value <= _UTF8_TWO_BYTE_MAX) { | |
| 96 encoded[insertAt] = _UTF8_FIRST_BYTE_OF_TWO_BASE | ( | |
| 97 _UTF8_FIRST_BYTE_OF_TWO_MASK & | |
| 98 _addToEncoding(insertAt, 1, value, encoded)); | |
| 99 insertAt += 2; | |
| 100 } else if (value <= _UTF8_THREE_BYTE_MAX) { | |
| 101 encoded[insertAt] = _UTF8_FIRST_BYTE_OF_THREE_BASE | ( | |
| 102 _UTF8_FIRST_BYTE_OF_THREE_MASK & | |
| 103 _addToEncoding(insertAt, 2, value, encoded)); | |
| 104 insertAt += 3; | |
| 105 } else if (value <= UNICODE_VALID_RANGE_MAX) { | |
| 106 encoded[insertAt] = _UTF8_FIRST_BYTE_OF_FOUR_BASE | ( | |
| 107 _UTF8_FIRST_BYTE_OF_FOUR_MASK & | |
| 108 _addToEncoding(insertAt, 3, value, encoded)); | |
| 109 insertAt += 4; | |
| 110 } | |
| 111 } | |
| 112 return encoded; | |
| 113 } | |
| 114 | |
| 115 | |
| 116 // Because UTF-8 specifies byte order, we do not have to follow the pattern | |
| 117 // used by UTF-16 & UTF-32 regarding byte order. | |
| 118 List<int> _utf8ToCodepoints( | |
| 119 List<int> utf8EncodedBytes, [int offset = 0, int length, | |
| 120 int replacementCodepoint = UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]) { | |
| 121 if (!(offset >= 0)) { | |
| 122 throw new IllegalArgumentException("offset"); | |
| 123 } | |
| 124 | |
| 125 if (!(length == null || length >= 0)) { | |
| 126 throw new IllegalArgumentException("length"); | |
| 127 } | |
| 128 | |
| 129 int end = length != null ? | |
| 130 Math.min(utf8EncodedBytes.length, offset + length) : | |
| 131 utf8EncodedBytes.length; | |
| 132 | |
| 133 void addReplacementCodepoint(void f(int v), int replacementCodepoint) { | |
| 134 if(replacementCodepoint != null) { | |
| 135 f(replacementCodepoint); | |
| 136 } else { | |
| 137 throw new IllegalArgumentException("Invalid encoding"); | |
| 138 } | |
| 139 } | |
| 140 | |
| 141 void apply(void f(int v)) { | |
| 142 int i = offset; | |
| 143 while (i < end) { | |
| 144 int value = utf8EncodedBytes[i++]; | |
| 145 if (value >= 0) { | |
| 146 if (value <= _UTF8_ONE_BYTE_MAX) { | |
| 147 f(value); | |
| 148 } else if (value < _UTF8_FIRST_BYTE_OF_TWO_BASE) { | |
| 149 addReplacementCodepoint(f, replacementCodepoint); | |
| 150 continue; | |
| 151 } else { | |
| 152 int additionalBytes = 0; | |
| 153 if (value < _UTF8_FIRST_BYTE_OF_THREE_BASE) { | |
| 154 value -= _UTF8_FIRST_BYTE_OF_TWO_BASE; | |
| 155 additionalBytes = 1; | |
| 156 } else if (value < _UTF8_FIRST_BYTE_OF_FOUR_BASE) { | |
| 157 value -= _UTF8_FIRST_BYTE_OF_THREE_BASE; | |
| 158 additionalBytes = 2; | |
| 159 } else if (value < _UTF8_FIRST_BYTE_OF_FIVE_BASE) { | |
| 160 value -= _UTF8_FIRST_BYTE_OF_FOUR_BASE; | |
| 161 additionalBytes = 3; | |
| 162 } else if (value < _UTF8_FIRST_BYTE_OF_SIX_BASE) { | |
| 163 value -= _UTF8_FIRST_BYTE_OF_FIVE_BASE; | |
| 164 additionalBytes = 4; | |
| 165 } else if (value < _UTF8_FIRST_BYTE_BOUND_EXCL) { | |
| 166 value -= _UTF8_FIRST_BYTE_OF_SIX_BASE; | |
| 167 additionalBytes = 5; | |
| 168 } else { | |
| 169 addReplacementCodepoint(f, replacementCodepoint); | |
| 170 continue; | |
| 171 } | |
| 172 int j = 0; | |
| 173 while (j < additionalBytes && i < end) { | |
| 174 int nextValue = utf8EncodedBytes[i++]; | |
| 175 if (nextValue > _UTF8_ONE_BYTE_MAX && | |
| 176 nextValue < _UTF8_FIRST_BYTE_OF_TWO_BASE) { | |
| 177 value = (value << 6) | (nextValue & _UTF8_LO_SIX_BIT_MASK); | |
| 178 } else { | |
| 179 // if sequence-starting code unit, reposition cursor to start here | |
| 180 if (nextValue >= _UTF8_FIRST_BYTE_OF_TWO_BASE) { | |
| 181 i--; | |
| 182 } | |
| 183 break; | |
| 184 } | |
| 185 j++; | |
| 186 } | |
| 187 if (j == additionalBytes && ( | |
| 188 value < UNICODE_UTF16_RESERVED_LO || | |
| 189 value > UNICODE_UTF16_RESERVED_HI)) { | |
| 190 if ((additionalBytes == 1 && value > _UTF8_ONE_BYTE_MAX) || | |
| 191 (additionalBytes == 2 && value > _UTF8_TWO_BYTE_MAX) || | |
| 192 (additionalBytes == 3 && value > _UTF8_THREE_BYTE_MAX && | |
| 193 value <= UNICODE_VALID_RANGE_MAX)) { | |
| 194 f(value); | |
| 195 } else { | |
| 196 addReplacementCodepoint(f, replacementCodepoint); | |
| 197 } | |
| 198 } else { | |
| 199 addReplacementCodepoint(f, replacementCodepoint); | |
| 200 continue; | |
| 201 } | |
| 202 } | |
| 203 } else { | |
| 204 addReplacementCodepoint(f, replacementCodepoint); | |
| 205 continue; | |
| 206 } | |
| 207 } | |
| 208 } | |
| 209 | |
| 210 int codepointBufferLength = 0; | |
| 211 apply(void _(int value) { | |
| 212 codepointBufferLength++; | |
| 213 }); | |
| 214 | |
| 215 List<int> codepointBuffer = new List<int>(codepointBufferLength); | |
| 216 int i = 0; | |
| 217 apply(void _(int value) { | |
| 218 codepointBuffer[i++] = value; | |
| 219 }); | |
| 220 return codepointBuffer; | |
| 221 } | |
| OLD | NEW |