Scanner can include comments in the token stream.

lib/compiler/implementation/scanner/token.dart

 // Copyright (c) 2011, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 final int EOF_TOKEN = 0;
 
 final int KEYWORD_TOKEN = $k;
 final int IDENTIFIER_TOKEN = $a;
 final int DOUBLE_TOKEN = $d;
 final int INT_TOKEN = $i;
@@ skipping 53 matching lines @@
 final int PLUS_PLUS_TOKEN = STAR_EQ_TOKEN + 1;
 final int PLUS_EQ_TOKEN = PLUS_PLUS_TOKEN + 1;
 final int MINUS_MINUS_TOKEN = PLUS_EQ_TOKEN + 1;
 final int MINUS_EQ_TOKEN = MINUS_MINUS_TOKEN + 1;
 final int TILDE_SLASH_EQ_TOKEN = MINUS_EQ_TOKEN + 1;
 final int TILDE_SLASH_TOKEN = TILDE_SLASH_EQ_TOKEN + 1;
 final int PERCENT_EQ_TOKEN = TILDE_SLASH_TOKEN + 1;
 final int GT_GT_TOKEN = PERCENT_EQ_TOKEN + 1;
 final int CARET_EQ_TOKEN = GT_GT_TOKEN + 1;
 final int IS_TOKEN = CARET_EQ_TOKEN + 1;
+final int COMMENT_TOKEN = IS_TOKEN + 1;
+final int GT_GT_GT_TOKEN = COMMENT_TOKEN + 1;
+final int STRING_INTERPOLATION_IDENTIFIER_TOKEN = COMMENT_TOKEN + 1;
 
 // TODO(ahe): Get rid of this.
 final int UNKNOWN_TOKEN = 1024;
 
 /**
  * A token that doubles as a linked list.
  */
 class Token {
+  /**
+   * The precedence info for this token. [info] determines the kind and the
+   * precedence level of this token.
+   */
   final PrecedenceInfo info;
+
+  /**
+   * The character offset of the start of this token within the source text.
+   */
   final int charOffset;
+
+  /**
+   * The next token in the token stream.
+   */
   Token next;
 
   Token(PrecedenceInfo this.info, int this.charOffset);
 
   get value() => info.value;
+
+  /**
+   * Returns the string value for keywords and symbols. For instance 'class' for
+   * the [CLASS] keyword token and '*' for a [Token] based on [STAR_INFO]. For
+   * other tokens, such identifiers, strings, numbers, etc, [stringValue]
+   * returns [:null:].
+   *
+   * [stringValue] should only be used for testing keywords and symbols.
+   */
   String get stringValue() => info.value.stringValue;
+
+  /**
+   * The kind enum of this token as determined by its [info].
+   */
   int get kind() => info.kind;
+
+  /**
+   * The precedence level for this token.
+   */
   int get precedence() => info.precedence;
 
+  /**
+   * Returns a textual representation of this token to be used for debugging
+   * purposes. The resulting string might contain information about the
+   * structure of the token, for example 'StringToken(foo)' for the identifier
+   * token 'foo'. Use [slowToString] for the text actually parsed by the token.
+   */
   String toString() => info.value.toString();
 
+  /**
+   * The text parsed by this token.
+   */
   String slowToString() => toString();
+
+  /**
+   * The number of characters parsed by this token.
+   */
+  int get slowCharLength() => slowToString().length;
 }
 
 /**
  * A keyword token.
  */
 class KeywordToken extends Token {
   final Keyword value;
   String get stringValue() => value.syntax;
 
   KeywordToken(Keyword value, int charOffset)
@@ skipping 254 matching lines @@
   const PrecedenceInfo(const SourceString('>'), 10, GT_TOKEN);
 final PrecedenceInfo IS_INFO =
   const PrecedenceInfo(const SourceString('is'), 10, IS_TOKEN);
 final PrecedenceInfo LT_EQ_INFO =
   const PrecedenceInfo(const SourceString('<='), 10, LT_EQ_TOKEN);
 final PrecedenceInfo LT_INFO =
   const PrecedenceInfo(const SourceString('<'), 10, LT_TOKEN);
 
 // Shift operators.
 final PrecedenceInfo GT_GT_GT_INFO =
-  const PrecedenceInfo(const SourceString('>>>'), 11, GT_GT_TOKEN);
+  const PrecedenceInfo(const SourceString('>>>'), 11, GT_GT_GT_TOKEN);
 final PrecedenceInfo GT_GT_INFO =
   const PrecedenceInfo(const SourceString('>>'), 11, GT_GT_TOKEN);
 final PrecedenceInfo LT_LT_INFO =
   const PrecedenceInfo(const SourceString('<<'), 11, LT_LT_TOKEN);
 
 // Additive operators.
 final PrecedenceInfo MINUS_INFO =
   const PrecedenceInfo(const SourceString('-'), 12, MINUS_TOKEN);
 final PrecedenceInfo PLUS_INFO =
   const PrecedenceInfo(const SourceString('+'), 12, PLUS_TOKEN);
@@ skipping 48 matching lines @@
 final PrecedenceInfo CLOSE_SQUARE_BRACKET_INFO =
   const PrecedenceInfo(const SourceString(']'), 0, CLOSE_SQUARE_BRACKET_TOKEN);
 
 final PrecedenceInfo DOUBLE_INFO =
   const PrecedenceInfo(const SourceString('double'), 0, DOUBLE_TOKEN);
 
 final PrecedenceInfo STRING_INTERPOLATION_INFO =
   const PrecedenceInfo(const SourceString('\${'), 0,
                        STRING_INTERPOLATION_TOKEN);
 
+final PrecedenceInfo STRING_INTERPOLATION_IDENTIFIER_INFO =
+  const PrecedenceInfo(const SourceString('\$'), 0,
+                       STRING_INTERPOLATION_IDENTIFIER_TOKEN);
+
 final PrecedenceInfo HEXADECIMAL_INFO =
   const PrecedenceInfo(const SourceString('hexadecimal'), 0, HEXADECIMAL_TOKEN);
 
+final PrecedenceInfo COMMENT_INFO =
+  const PrecedenceInfo(const SourceString('comment'), 0, COMMENT_TOKEN);
+
 // For reporting lexical errors.
 final PrecedenceInfo ERROR_INFO =
   const PrecedenceInfo(const SourceString('?'), 0, UNKNOWN_TOKEN);