Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(104)

Side by Side Diff: frog/leg/scanner/token.dart

Issue 9873021: Move frog/leg to lib/compiler/implementation. (Closed) Base URL: http://dart.googlecode.com/svn/branches/bleeding_edge/dart/
Patch Set: Created 8 years, 9 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch | Annotate | Revision Log
« no previous file with comments | « frog/leg/scanner/string_scanner.dart ('k') | frog/leg/scanner/vm_scanner_bench.dart » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
(Empty)
1 // Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file
2 // for details. All rights reserved. Use of this source code is governed by a
3 // BSD-style license that can be found in the LICENSE file.
4
5 final int EOF_TOKEN = 0;
6
7 final int KEYWORD_TOKEN = $k;
8 final int IDENTIFIER_TOKEN = $a;
9 final int DOUBLE_TOKEN = $d;
10 final int INT_TOKEN = $i;
11 final int HEXADECIMAL_TOKEN = $x;
12 final int STRING_TOKEN = $SQ;
13
14 final int AMPERSAND_TOKEN = $AMPERSAND;
15 final int BACKPING_TOKEN = $BACKPING;
16 final int BACKSLASH_TOKEN = $BACKSLASH;
17 final int BANG_TOKEN = $BANG;
18 final int BAR_TOKEN = $BAR;
19 final int COLON_TOKEN = $COLON;
20 final int COMMA_TOKEN = $COMMA;
21 final int EQ_TOKEN = $EQ;
22 final int GT_TOKEN = $GT;
23 final int HASH_TOKEN = $HASH;
24 final int OPEN_CURLY_BRACKET_TOKEN = $OPEN_CURLY_BRACKET;
25 final int OPEN_SQUARE_BRACKET_TOKEN = $OPEN_SQUARE_BRACKET;
26 final int OPEN_PAREN_TOKEN = $OPEN_PAREN;
27 final int LT_TOKEN = $LT;
28 final int MINUS_TOKEN = $MINUS;
29 final int PERIOD_TOKEN = $PERIOD;
30 final int PLUS_TOKEN = $PLUS;
31 final int QUESTION_TOKEN = $QUESTION;
32 final int CLOSE_CURLY_BRACKET_TOKEN = $CLOSE_CURLY_BRACKET;
33 final int CLOSE_SQUARE_BRACKET_TOKEN = $CLOSE_SQUARE_BRACKET;
34 final int CLOSE_PAREN_TOKEN = $CLOSE_PAREN;
35 final int SEMICOLON_TOKEN = $SEMICOLON;
36 final int SLASH_TOKEN = $SLASH;
37 final int TILDE_TOKEN = $TILDE;
38 final int STAR_TOKEN = $STAR;
39 final int PERCENT_TOKEN = $PERCENT;
40 final int CARET_TOKEN = $CARET;
41
42 final int STRING_INTERPOLATION_TOKEN = 128;
43 final int LT_EQ_TOKEN = STRING_INTERPOLATION_TOKEN + 1;
44 final int FUNCTION_TOKEN = LT_EQ_TOKEN + 1;
45 final int SLASH_EQ_TOKEN = FUNCTION_TOKEN + 1;
46 final int PERIOD_PERIOD_PERIOD_TOKEN = SLASH_EQ_TOKEN + 1;
47 final int PERIOD_PERIOD_TOKEN = PERIOD_PERIOD_PERIOD_TOKEN + 1;
48 final int EQ_EQ_EQ_TOKEN = PERIOD_PERIOD_TOKEN + 1;
49 final int EQ_EQ_TOKEN = EQ_EQ_EQ_TOKEN + 1;
50 final int LT_LT_EQ_TOKEN = EQ_EQ_TOKEN + 1;
51 final int LT_LT_TOKEN = LT_LT_EQ_TOKEN + 1;
52 final int GT_EQ_TOKEN = LT_LT_TOKEN + 1;
53 final int GT_GT_EQ_TOKEN = GT_EQ_TOKEN + 1;
54 final int GT_GT_GT_EQ_TOKEN = GT_GT_EQ_TOKEN + 1;
55 final int INDEX_EQ_TOKEN = GT_GT_GT_EQ_TOKEN + 1;
56 final int INDEX_TOKEN = INDEX_EQ_TOKEN + 1;
57 final int BANG_EQ_EQ_TOKEN = INDEX_TOKEN + 1;
58 final int BANG_EQ_TOKEN = BANG_EQ_EQ_TOKEN + 1;
59 final int AMPERSAND_AMPERSAND_TOKEN = BANG_EQ_TOKEN + 1;
60 final int AMPERSAND_EQ_TOKEN = AMPERSAND_AMPERSAND_TOKEN + 1;
61 final int BAR_BAR_TOKEN = AMPERSAND_EQ_TOKEN + 1;
62 final int BAR_EQ_TOKEN = BAR_BAR_TOKEN + 1;
63 final int STAR_EQ_TOKEN = BAR_EQ_TOKEN + 1;
64 final int PLUS_PLUS_TOKEN = STAR_EQ_TOKEN + 1;
65 final int PLUS_EQ_TOKEN = PLUS_PLUS_TOKEN + 1;
66 final int MINUS_MINUS_TOKEN = PLUS_EQ_TOKEN + 1;
67 final int MINUS_EQ_TOKEN = MINUS_MINUS_TOKEN + 1;
68 final int TILDE_SLASH_EQ_TOKEN = MINUS_EQ_TOKEN + 1;
69 final int TILDE_SLASH_TOKEN = TILDE_SLASH_EQ_TOKEN + 1;
70 final int PERCENT_EQ_TOKEN = TILDE_SLASH_TOKEN + 1;
71 final int GT_GT_TOKEN = PERCENT_EQ_TOKEN + 1;
72 final int CARET_EQ_TOKEN = GT_GT_TOKEN + 1;
73 final int IS_TOKEN = CARET_EQ_TOKEN + 1;
74
75 // TODO(ahe): Get rid of this.
76 final int UNKNOWN_TOKEN = 1024;
77
78 /**
79 * A token that doubles as a linked list.
80 */
81 class Token {
82 final PrecedenceInfo info;
83 final int charOffset;
84 Token next;
85
86 Token(PrecedenceInfo this.info, int this.charOffset);
87
88 get value() => info.value;
89 String get stringValue() => info.value.stringValue;
90 int get kind() => info.kind;
91 int get precedence() => info.precedence;
92
93 String toString() => info.value.toString();
94
95 String slowToString() => toString();
96 }
97
98 /**
99 * A keyword token.
100 */
101 class KeywordToken extends Token {
102 final Keyword value;
103 String get stringValue() => value.syntax;
104
105 KeywordToken(Keyword value, int charOffset)
106 : this.value = value, super(value.info, charOffset);
107
108 String toString() => value.syntax;
109 }
110
111 /**
112 * A String-valued token.
113 */
114 class StringToken extends Token {
115 final SourceString value;
116 String get stringValue() => value.stringValue;
117
118 StringToken(PrecedenceInfo info, String value, int charOffset)
119 : this.fromSource(info, new SourceString(value), charOffset);
120
121 StringToken.fromSource(PrecedenceInfo info, this.value, int charOffset)
122 : super(info, charOffset);
123
124 String toString() => "StringToken(${value.slowToString()})";
125
126 String slowToString() => value.slowToString();
127 }
128
129 interface SourceString extends Hashable, Iterable<int> default StringWrapper {
130 const SourceString(String string);
131
132 void printOn(StringBuffer sb);
133
134 /** Gives a [SourceString] that is not including the [initial] first and
135 * [terminal] last characters. This is only intended to be used to remove
136 * quotes from string literals (including an initial '@' for raw strings).
137 */
138 SourceString copyWithoutQuotes(int initial, int terminal);
139
140 String get stringValue();
141
142 String slowToString();
143
144 bool isEmpty();
145
146 bool isPrivate();
147 }
148
149 class StringWrapper implements SourceString {
150 final String stringValue;
151
152 const StringWrapper(String this.stringValue);
153
154 int hashCode() => stringValue.hashCode();
155
156 bool operator ==(other) {
157 return other is SourceString && toString() == other.slowToString();
158 }
159
160 Iterator<int> iterator() => new StringCodeIterator(stringValue);
161
162 void printOn(StringBuffer sb) {
163 sb.add(stringValue);
164 }
165
166 String toString() => stringValue;
167
168 String slowToString() => stringValue;
169
170 SourceString copyWithoutQuotes(int initial, int terminal) {
171 assert(0 <= initial);
172 assert(0 <= terminal);
173 assert(initial + terminal <= stringValue.length);
174 return new StringWrapper(
175 stringValue.substring(initial, stringValue.length - terminal));
176 }
177
178 bool isEmpty() => stringValue.isEmpty();
179
180 bool isPrivate() => !isEmpty() && stringValue.charCodeAt(0) === $_;
181 }
182
183 class StringCodeIterator implements Iterator<int> {
184 final String string;
185 int index;
186 final int end;
187
188 StringCodeIterator(String string) :
189 this.string = string, index = 0, end = string.length;
190
191 StringCodeIterator.substring(this.string, this.index, this.end) {
192 assert(0 <= index);
193 assert(index <= end);
194 assert(end <= string.length);
195 }
196
197 bool hasNext() => index < end;
198 int next() => string.charCodeAt(index++);
199 }
200
201 class BeginGroupToken extends StringToken {
202 Token endGroup;
203 BeginGroupToken(PrecedenceInfo info, String value, int charOffset)
204 : super(info, value, charOffset);
205 }
206
207 bool isUserDefinableOperator(String value) {
208 return
209 (value === '==') ||
210 (value === '~') ||
211 (value === 'negate') ||
212 (value === '[]') ||
213 (value === '[]=') ||
214 (value === '*') ||
215 (value === '/') ||
216 (value === '%') ||
217 (value === '~/') ||
218 (value === '+') ||
219 (value === '-') ||
220 (value === '<<') ||
221 (value === '>>>') ||
222 (value === '>>') ||
223 (value === '>=') ||
224 (value === '>') ||
225 (value === '<=') ||
226 (value === '<') ||
227 (value === '&') ||
228 (value === '^') ||
229 (value === '|');
230 }
231
232 class PrecedenceInfo {
233 final SourceString value;
234 final int precedence;
235 final int kind;
236
237 const PrecedenceInfo(this.value, this.precedence, this.kind);
238
239 toString() => 'PrecedenceInfo($value, $precedence, $kind)';
240 }
241
242 // TODO(ahe): The following are not tokens in Dart.
243 final PrecedenceInfo BACKPING_INFO =
244 const PrecedenceInfo(const SourceString('`'), 0, BACKPING_TOKEN);
245 final PrecedenceInfo BACKSLASH_INFO =
246 const PrecedenceInfo(const SourceString('\\'), 0, BACKSLASH_TOKEN);
247 final PrecedenceInfo PERIOD_PERIOD_PERIOD_INFO =
248 const PrecedenceInfo(const SourceString('...'), 0,
249 PERIOD_PERIOD_PERIOD_TOKEN);
250
251 // TODO(ahe): This might become a token.
252 final PrecedenceInfo PERIOD_PERIOD_INFO =
253 const PrecedenceInfo(const SourceString('..'), 0, PERIOD_PERIOD_TOKEN);
254
255 final PrecedenceInfo BANG_INFO =
256 const PrecedenceInfo(const SourceString('!'), 0, BANG_TOKEN);
257 final PrecedenceInfo COLON_INFO =
258 const PrecedenceInfo(const SourceString(':'), 0, COLON_TOKEN);
259 final PrecedenceInfo INDEX_INFO =
260 const PrecedenceInfo(const SourceString('[]'), 0, INDEX_TOKEN);
261 final PrecedenceInfo MINUS_MINUS_INFO =
262 const PrecedenceInfo(const SourceString('--'), POSTFIX_PRECEDENCE,
263 MINUS_MINUS_TOKEN);
264 final PrecedenceInfo PLUS_PLUS_INFO =
265 const PrecedenceInfo(const SourceString('++'), POSTFIX_PRECEDENCE,
266 PLUS_PLUS_TOKEN);
267 final PrecedenceInfo TILDE_INFO =
268 const PrecedenceInfo(const SourceString('~'), 0, TILDE_TOKEN);
269
270 final PrecedenceInfo FUNCTION_INFO =
271 const PrecedenceInfo(const SourceString('=>'), 0, FUNCTION_TOKEN);
272 final PrecedenceInfo HASH_INFO =
273 const PrecedenceInfo(const SourceString('#'), 0, HASH_TOKEN);
274 final PrecedenceInfo INDEX_EQ_INFO =
275 const PrecedenceInfo(const SourceString('[]='), 0, INDEX_EQ_TOKEN);
276 final PrecedenceInfo SEMICOLON_INFO =
277 const PrecedenceInfo(const SourceString(';'), 0, SEMICOLON_TOKEN);
278 final PrecedenceInfo COMMA_INFO =
279 const PrecedenceInfo(const SourceString(','), 0, COMMA_TOKEN);
280
281 // Assignment operators.
282 final int ASSIGNMENT_PRECEDENCE = 2;
283 final PrecedenceInfo AMPERSAND_EQ_INFO =
284 const PrecedenceInfo(const SourceString('&='),
285 ASSIGNMENT_PRECEDENCE, AMPERSAND_EQ_TOKEN);
286 final PrecedenceInfo BAR_EQ_INFO =
287 const PrecedenceInfo(const SourceString('|='),
288 ASSIGNMENT_PRECEDENCE, BAR_EQ_TOKEN);
289 final PrecedenceInfo CARET_EQ_INFO =
290 const PrecedenceInfo(const SourceString('^='),
291 ASSIGNMENT_PRECEDENCE, CARET_EQ_TOKEN);
292 final PrecedenceInfo EQ_INFO =
293 const PrecedenceInfo(const SourceString('='),
294 ASSIGNMENT_PRECEDENCE, EQ_TOKEN);
295 final PrecedenceInfo GT_GT_EQ_INFO =
296 const PrecedenceInfo(const SourceString('>>='),
297 ASSIGNMENT_PRECEDENCE, GT_GT_EQ_TOKEN);
298 final PrecedenceInfo GT_GT_GT_EQ_INFO =
299 const PrecedenceInfo(const SourceString('>>>='),
300 ASSIGNMENT_PRECEDENCE, GT_GT_GT_EQ_TOKEN);
301 final PrecedenceInfo LT_LT_EQ_INFO =
302 const PrecedenceInfo(const SourceString('<<='),
303 ASSIGNMENT_PRECEDENCE, LT_LT_EQ_TOKEN);
304 final PrecedenceInfo MINUS_EQ_INFO =
305 const PrecedenceInfo(const SourceString('-='),
306 ASSIGNMENT_PRECEDENCE, MINUS_EQ_TOKEN);
307 final PrecedenceInfo PERCENT_EQ_INFO =
308 const PrecedenceInfo(const SourceString('%='),
309 ASSIGNMENT_PRECEDENCE, PERCENT_EQ_TOKEN);
310 final PrecedenceInfo PLUS_EQ_INFO =
311 const PrecedenceInfo(const SourceString('+='),
312 ASSIGNMENT_PRECEDENCE, PLUS_EQ_TOKEN);
313 final PrecedenceInfo SLASH_EQ_INFO =
314 const PrecedenceInfo(const SourceString('/='),
315 ASSIGNMENT_PRECEDENCE, SLASH_EQ_TOKEN);
316 final PrecedenceInfo STAR_EQ_INFO =
317 const PrecedenceInfo(const SourceString('*='),
318 ASSIGNMENT_PRECEDENCE, STAR_EQ_TOKEN);
319 final PrecedenceInfo TILDE_SLASH_EQ_INFO =
320 const PrecedenceInfo(const SourceString('~/='),
321 ASSIGNMENT_PRECEDENCE, TILDE_SLASH_EQ_TOKEN);
322
323 final PrecedenceInfo QUESTION_INFO =
324 const PrecedenceInfo(const SourceString('?'), 3, QUESTION_TOKEN);
325
326 final PrecedenceInfo BAR_BAR_INFO =
327 const PrecedenceInfo(const SourceString('||'), 4, BAR_BAR_TOKEN);
328
329 final PrecedenceInfo AMPERSAND_AMPERSAND_INFO =
330 const PrecedenceInfo(const SourceString('&&'), 5, AMPERSAND_AMPERSAND_TOKEN);
331
332 final PrecedenceInfo BAR_INFO =
333 const PrecedenceInfo(const SourceString('|'), 6, BAR_TOKEN);
334
335 final PrecedenceInfo CARET_INFO =
336 const PrecedenceInfo(const SourceString('^'), 7, CARET_TOKEN);
337
338 final PrecedenceInfo AMPERSAND_INFO =
339 const PrecedenceInfo(const SourceString('&'), 8, AMPERSAND_TOKEN);
340
341 // Equality operators.
342 final PrecedenceInfo BANG_EQ_EQ_INFO =
343 const PrecedenceInfo(const SourceString('!=='), 9, BANG_EQ_EQ_TOKEN);
344 final PrecedenceInfo BANG_EQ_INFO =
345 const PrecedenceInfo(const SourceString('!='), 9, BANG_EQ_TOKEN);
346 final PrecedenceInfo EQ_EQ_EQ_INFO =
347 const PrecedenceInfo(const SourceString('==='), 9, EQ_EQ_EQ_TOKEN);
348 final PrecedenceInfo EQ_EQ_INFO =
349 const PrecedenceInfo(const SourceString('=='), 9, EQ_EQ_TOKEN);
350
351 // Relational operators.
352 final PrecedenceInfo GT_EQ_INFO =
353 const PrecedenceInfo(const SourceString('>='), 10, GT_EQ_TOKEN);
354 final PrecedenceInfo GT_INFO =
355 const PrecedenceInfo(const SourceString('>'), 10, GT_TOKEN);
356 final PrecedenceInfo IS_INFO =
357 const PrecedenceInfo(const SourceString('is'), 10, IS_TOKEN);
358 final PrecedenceInfo LT_EQ_INFO =
359 const PrecedenceInfo(const SourceString('<='), 10, LT_EQ_TOKEN);
360 final PrecedenceInfo LT_INFO =
361 const PrecedenceInfo(const SourceString('<'), 10, LT_TOKEN);
362
363 // Shift operators.
364 final PrecedenceInfo GT_GT_GT_INFO =
365 const PrecedenceInfo(const SourceString('>>>'), 11, GT_GT_TOKEN);
366 final PrecedenceInfo GT_GT_INFO =
367 const PrecedenceInfo(const SourceString('>>'), 11, GT_GT_TOKEN);
368 final PrecedenceInfo LT_LT_INFO =
369 const PrecedenceInfo(const SourceString('<<'), 11, LT_LT_TOKEN);
370
371 // Additive operators.
372 final PrecedenceInfo MINUS_INFO =
373 const PrecedenceInfo(const SourceString('-'), 12, MINUS_TOKEN);
374 final PrecedenceInfo PLUS_INFO =
375 const PrecedenceInfo(const SourceString('+'), 12, PLUS_TOKEN);
376
377 // Multiplicative operators.
378 final PrecedenceInfo PERCENT_INFO =
379 const PrecedenceInfo(const SourceString('%'), 13, PERCENT_TOKEN);
380 final PrecedenceInfo SLASH_INFO =
381 const PrecedenceInfo(const SourceString('/'), 13, SLASH_TOKEN);
382 final PrecedenceInfo STAR_INFO =
383 const PrecedenceInfo(const SourceString('*'), 13, STAR_TOKEN);
384 final PrecedenceInfo TILDE_SLASH_INFO =
385 const PrecedenceInfo(const SourceString('~/'), 13, TILDE_SLASH_TOKEN);
386
387 final int POSTFIX_PRECEDENCE = 14;
388 final PrecedenceInfo PERIOD_INFO =
389 const PrecedenceInfo(const SourceString('.'), POSTFIX_PRECEDENCE,
390 PERIOD_TOKEN);
391
392 final PrecedenceInfo KEYWORD_INFO =
393 const PrecedenceInfo(const SourceString('keyword'), 0, KEYWORD_TOKEN);
394
395 final PrecedenceInfo EOF_INFO =
396 const PrecedenceInfo(const SourceString('EOF'), 0, EOF_TOKEN);
397
398 final PrecedenceInfo IDENTIFIER_INFO =
399 const PrecedenceInfo(const SourceString('identifier'), 0, IDENTIFIER_TOKEN);
400
401 final PrecedenceInfo OPEN_PAREN_INFO =
402 const PrecedenceInfo(const SourceString('('), POSTFIX_PRECEDENCE,
403 OPEN_PAREN_TOKEN);
404
405 final PrecedenceInfo CLOSE_PAREN_INFO =
406 const PrecedenceInfo(const SourceString(')'), 0, CLOSE_PAREN_TOKEN);
407
408 final PrecedenceInfo OPEN_CURLY_BRACKET_INFO =
409 const PrecedenceInfo(const SourceString('{'), 0, OPEN_CURLY_BRACKET_TOKEN);
410
411 final PrecedenceInfo CLOSE_CURLY_BRACKET_INFO =
412 const PrecedenceInfo(const SourceString('}'), 0, CLOSE_CURLY_BRACKET_TOKEN);
413
414 final PrecedenceInfo INT_INFO =
415 const PrecedenceInfo(const SourceString('int'), 0, INT_TOKEN);
416
417 final PrecedenceInfo STRING_INFO =
418 const PrecedenceInfo(const SourceString('string'), 0, STRING_TOKEN);
419
420 final PrecedenceInfo OPEN_SQUARE_BRACKET_INFO =
421 const PrecedenceInfo(const SourceString('['), POSTFIX_PRECEDENCE,
422 OPEN_SQUARE_BRACKET_TOKEN);
423
424 final PrecedenceInfo CLOSE_SQUARE_BRACKET_INFO =
425 const PrecedenceInfo(const SourceString(']'), 0, CLOSE_SQUARE_BRACKET_TOKEN);
426
427 final PrecedenceInfo DOUBLE_INFO =
428 const PrecedenceInfo(const SourceString('double'), 0, DOUBLE_TOKEN);
429
430 final PrecedenceInfo STRING_INTERPOLATION_INFO =
431 const PrecedenceInfo(const SourceString('\${'), 0,
432 STRING_INTERPOLATION_TOKEN);
433
434 final PrecedenceInfo HEXADECIMAL_INFO =
435 const PrecedenceInfo(const SourceString('hexadecimal'), 0, HEXADECIMAL_TOKEN);
436
437 // For reporting lexical errors.
438 final PrecedenceInfo ERROR_INFO =
439 const PrecedenceInfo(const SourceString('?'), 0, UNKNOWN_TOKEN);
OLDNEW
« no previous file with comments | « frog/leg/scanner/string_scanner.dart ('k') | frog/leg/scanner/vm_scanner_bench.dart » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698