Represent tokens as a compressed stream instead of an array.

vm/parser.h

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 #ifndef VM_PARSER_H_
 #define VM_PARSER_H_
 
 #include "include/dart_api.h"
 
 #include "vm/ast.h"
@@ skipping 87 matching lines @@
 
   int first_parameter_index_;
   int first_stack_local_index_;
   int copied_parameter_count_;
   int stack_local_count_;
 
   DISALLOW_COPY_AND_ASSIGN(ParsedFunction);
 };
 
 
-// The class TokenStreamIterator encapsulates iteration over the TokenStream
-// object. The parser uses this to iterate over tokens while parsing a script
-// or a function.
-class TokenStreamIterator : ValueObject {
- public:
-  TokenStreamIterator(const TokenStream& tokens, intptr_t token_pos)
-      : tokens_(tokens), token_position_(token_pos) { }
-
-  intptr_t NumberOfTokens() const { return tokens_.Length(); }
-  bool IsValid() const;
-
-  inline Token::Kind CurrentTokenKind() const;
-  Token::Kind LookaheadTokenKind(intptr_t num_tokens) const;
-
-  intptr_t CurrentPosition() const { return token_position_; }
-  void SetCurrentPosition(intptr_t value) { token_position_ = value; }
-
-  void Advance() { token_position_ += 1; }
-
-  RawObject* CurrentToken() const;
-  RawString* CurrentLiteral() const;
-
- private:
-  const TokenStream& tokens_;
-  intptr_t token_position_;
-};
-
-
 class Parser : ValueObject {
  public:
   Parser(const Script& script, const Library& library);
   Parser(const Script& script, const Function& function, intptr_t token_pos);
 
   // Parse the top level of a whole script file and register declared classes
   // and interfaces in the given library.
   static void ParseCompilationUnit(const Library& library,
                                    const Script& script);
 
@@ skipping 387 matching lines @@
   AstNode* InsertClosureCallNodes(AstNode* condition);
 
   ConstructorCallNode* CreateConstructorCallNode(
       intptr_t token_pos,
       const AbstractTypeArguments& type_arguments,
       const Function& constructor,
       ArgumentListNode* arguments);
 
 
   const Script& script_;
-  TokenStreamIterator tokens_iterator_;
+  TokenStream::Iterator tokens_iterator_;
   Token::Kind token_kind_;  // Cached token kind for current token.
   Block* current_block_;
 
   // is_top_level_ is true if parsing the "top level" of a compilation unit,
   // that is interface and class definitions.
   bool is_top_level_;
 
   // The member currently being parsed during "top level" parsing.
   MemberDesc* current_member_;
 
@@ skipping 22 matching lines @@
 
   // Allocate temporary only once per function.
   LocalVariable* expression_temp_;
 
   DISALLOW_COPY_AND_ASSIGN(Parser);
 };
 
 }  // namespace dart
 
 #endif  // VM_PARSER_H_