Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(504)

Side by Side Diff: utils/string_encoding/Utf8_impl.dart

Issue 9310038: Fix mac/git renaming case failure. Aaarrrrgggghhhh. (Closed) Base URL: https://dart.googlecode.com/svn/branches/bleeding_edge/dart
Patch Set: Created 8 years, 10 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch | Annotate | Revision Log
« no previous file with comments | « utils/string_encoding/Utf8.dart ('k') | utils/string_encoding/unicode.dart » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
(Empty)
1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file
2 // for details. All rights reserved. Use of this source code is governed by a
3 // BSD-style license that can be found in the LICENSE file.
4
5 final int _UTF8_ONE_BYTE_MAX = 0x7f;
6 final int _UTF8_TWO_BYTE_MAX = 0x7ff;
7 final int _UTF8_THREE_BYTE_MAX = 0xffff;
8
9 final int _UTF8_LO_SIX_BIT_MASK = 0x3f;
10
11 final int _UTF8_FIRST_BYTE_OF_TWO_BASE = 0xc0;
12 final int _UTF8_FIRST_BYTE_OF_THREE_BASE = 0xe0;
13 final int _UTF8_FIRST_BYTE_OF_FOUR_BASE = 0xf0;
14 final int _UTF8_FIRST_BYTE_OF_FIVE_BASE = 0xf8;
15 final int _UTF8_FIRST_BYTE_OF_SIX_BASE = 0xfc;
16
17 final int _UTF8_FIRST_BYTE_OF_TWO_MASK = 0x1f;
18 final int _UTF8_FIRST_BYTE_OF_THREE_MASK = 0xf;
19 final int _UTF8_FIRST_BYTE_OF_FOUR_MASK = 0x7;
20
21 final int _UTF8_FIRST_BYTE_BOUND_EXCL = 0xfe;
22 final int _UTF8_SUBSEQUENT_BYTE_BASE = 0x80;
23
24 /**
25 * Produce a String from a sequence of UTF-8 encoded bytes. The parameters
26 * allow an offset into a list of bytes (as int), limiting the length of the
27 * values be decoded and the ability of override the default Unicode
28 * replacement character. Set the replacementCharacter to null to throw an
29 * IllegalArgumentException rather than replace the bad value.
30 */
31 String decodeFromUtf8(List<int> bytes, [int offset = 0, int length,
32 int replacementCodepoint = UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]) =>
33 codepointsToString(_utf8ToCodepoints(
34 bytes, offset, length, replacementCodepoint));
35
36 /**
37 * Produce a sequence of UTF-8 encoded bytes from the provided string.
38 */
39 List<int> encodeAsUtf8(String str) =>
40 _codepointsToUtf8(stringToCodepoints(str));
41
42 int _addToEncoding(int offset, int bytes, int value, List<int> buffer) {
43 while(bytes > 0) {
44 buffer[offset + bytes] = _UTF8_SUBSEQUENT_BYTE_BASE |
45 (value & _UTF8_LO_SIX_BIT_MASK);
46 value = value >> 6;
47 bytes--;
48 }
49 return value;
50 }
51
52 /**
53 * Encode code points as UTF-8 code units.
54 */
55 List<int> _codepointsToUtf8(
56 List<int> codepoints, [int offset = 0, int length]) {
57 if (!(offset >= 0)) {
58 throw new IllegalArgumentException("offset");
59 }
60
61 if (!(length == null || length >= 0)) {
62 throw new IllegalArgumentException("length");
63 }
64
65 int end = length != null ?
66 Math.min(codepoints.length, offset + length) :
67 codepoints.length;
68
69 int encodedLength = 0;
70 for (int i = offset; i < end; i++) {
71 int value = codepoints[i];
72 if (value < 0 || value > UNICODE_VALID_RANGE_MAX) {
73 encodedLength += 3;
74 } else if (value <= _UTF8_ONE_BYTE_MAX) {
75 encodedLength++;
76 } else if (value <= _UTF8_TWO_BYTE_MAX) {
77 encodedLength += 2;
78 } else if (value <= _UTF8_THREE_BYTE_MAX) {
79 encodedLength += 3;
80 } else if (value <= UNICODE_VALID_RANGE_MAX) {
81 encodedLength += 4;
82 }
83 }
84
85 List<int> encoded = new List<int>(encodedLength);
86 int insertAt = 0;
87 for (int i = offset; i < end; i++) {
88 int value = codepoints[i];
89 if (value < 0 || value > UNICODE_VALID_RANGE_MAX) {
90 encoded.setRange(insertAt, 3, [0xef, 0xbf, 0xbd]);
91 insertAt += 3;
92 } else if (value <= _UTF8_ONE_BYTE_MAX) {
93 encoded[insertAt] = value;
94 insertAt++;
95 } else if (value <= _UTF8_TWO_BYTE_MAX) {
96 encoded[insertAt] = _UTF8_FIRST_BYTE_OF_TWO_BASE | (
97 _UTF8_FIRST_BYTE_OF_TWO_MASK &
98 _addToEncoding(insertAt, 1, value, encoded));
99 insertAt += 2;
100 } else if (value <= _UTF8_THREE_BYTE_MAX) {
101 encoded[insertAt] = _UTF8_FIRST_BYTE_OF_THREE_BASE | (
102 _UTF8_FIRST_BYTE_OF_THREE_MASK &
103 _addToEncoding(insertAt, 2, value, encoded));
104 insertAt += 3;
105 } else if (value <= UNICODE_VALID_RANGE_MAX) {
106 encoded[insertAt] = _UTF8_FIRST_BYTE_OF_FOUR_BASE | (
107 _UTF8_FIRST_BYTE_OF_FOUR_MASK &
108 _addToEncoding(insertAt, 3, value, encoded));
109 insertAt += 4;
110 }
111 }
112 return encoded;
113 }
114
115
116 // Because UTF-8 specifies byte order, we do not have to follow the pattern
117 // used by UTF-16 & UTF-32 regarding byte order.
118 List<int> _utf8ToCodepoints(
119 List<int> utf8EncodedBytes, [int offset = 0, int length,
120 int replacementCodepoint = UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]) {
121 if (!(offset >= 0)) {
122 throw new IllegalArgumentException("offset");
123 }
124
125 if (!(length == null || length >= 0)) {
126 throw new IllegalArgumentException("length");
127 }
128
129 int end = length != null ?
130 Math.min(utf8EncodedBytes.length, offset + length) :
131 utf8EncodedBytes.length;
132
133 void addReplacementCodepoint(void f(int v), int replacementCodepoint) {
134 if(replacementCodepoint != null) {
135 f(replacementCodepoint);
136 } else {
137 throw new IllegalArgumentException("Invalid encoding");
138 }
139 }
140
141 void apply(void f(int v)) {
142 int i = offset;
143 while (i < end) {
144 int value = utf8EncodedBytes[i++];
145 if (value >= 0) {
146 if (value <= _UTF8_ONE_BYTE_MAX) {
147 f(value);
148 } else if (value < _UTF8_FIRST_BYTE_OF_TWO_BASE) {
149 addReplacementCodepoint(f, replacementCodepoint);
150 continue;
151 } else {
152 int additionalBytes = 0;
153 if (value < _UTF8_FIRST_BYTE_OF_THREE_BASE) {
154 value -= _UTF8_FIRST_BYTE_OF_TWO_BASE;
155 additionalBytes = 1;
156 } else if (value < _UTF8_FIRST_BYTE_OF_FOUR_BASE) {
157 value -= _UTF8_FIRST_BYTE_OF_THREE_BASE;
158 additionalBytes = 2;
159 } else if (value < _UTF8_FIRST_BYTE_OF_FIVE_BASE) {
160 value -= _UTF8_FIRST_BYTE_OF_FOUR_BASE;
161 additionalBytes = 3;
162 } else if (value < _UTF8_FIRST_BYTE_OF_SIX_BASE) {
163 value -= _UTF8_FIRST_BYTE_OF_FIVE_BASE;
164 additionalBytes = 4;
165 } else if (value < _UTF8_FIRST_BYTE_BOUND_EXCL) {
166 value -= _UTF8_FIRST_BYTE_OF_SIX_BASE;
167 additionalBytes = 5;
168 } else {
169 addReplacementCodepoint(f, replacementCodepoint);
170 continue;
171 }
172 int j = 0;
173 while (j < additionalBytes && i < end) {
174 int nextValue = utf8EncodedBytes[i++];
175 if (nextValue > _UTF8_ONE_BYTE_MAX &&
176 nextValue < _UTF8_FIRST_BYTE_OF_TWO_BASE) {
177 value = (value << 6) | (nextValue & _UTF8_LO_SIX_BIT_MASK);
178 } else {
179 // if sequence-starting code unit, reposition cursor to start here
180 if (nextValue >= _UTF8_FIRST_BYTE_OF_TWO_BASE) {
181 i--;
182 }
183 break;
184 }
185 j++;
186 }
187 if (j == additionalBytes && (
188 value < UNICODE_UTF16_RESERVED_LO ||
189 value > UNICODE_UTF16_RESERVED_HI)) {
190 if ((additionalBytes == 1 && value > _UTF8_ONE_BYTE_MAX) ||
191 (additionalBytes == 2 && value > _UTF8_TWO_BYTE_MAX) ||
192 (additionalBytes == 3 && value > _UTF8_THREE_BYTE_MAX &&
193 value <= UNICODE_VALID_RANGE_MAX)) {
194 f(value);
195 } else {
196 addReplacementCodepoint(f, replacementCodepoint);
197 }
198 } else {
199 addReplacementCodepoint(f, replacementCodepoint);
200 continue;
201 }
202 }
203 } else {
204 addReplacementCodepoint(f, replacementCodepoint);
205 continue;
206 }
207 }
208 }
209
210 int codepointBufferLength = 0;
211 apply(void _(int value) {
212 codepointBufferLength++;
213 });
214
215 List<int> codepointBuffer = new List<int>(codepointBufferLength);
216 int i = 0;
217 apply(void _(int value) {
218 codepointBuffer[i++] = value;
219 });
220 return codepointBuffer;
221 }
OLDNEW
« no previous file with comments | « utils/string_encoding/Utf8.dart ('k') | utils/string_encoding/unicode.dart » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698