Represent tokens as a compressed stream instead of an array.

vm/parser.cc

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 #include "vm/parser.h"
 
 #include "vm/bigint_operations.h"
 #include "vm/class_finalizer.h"
 #include "vm/compiler.h"
 #include "vm/compiler_stats.h"
@@ skipping 178 matching lines @@
     scope->AddVariable(context_var);
     set_saved_context_var(context_var);
   }
 
   // Frame indices are relative to the frame pointer and are decreasing.
   ASSERT(next_free_frame_index <= first_stack_local_index_);
   stack_local_count_ = first_stack_local_index_ - next_free_frame_index;
 }
 
 
-bool TokenStreamIterator::IsValid() const {
-  return !tokens_.IsNull();
-}
-
-
-Token::Kind TokenStreamIterator::CurrentTokenKind() const {
-  return tokens_.KindAt(token_position_);
-}
-
-
-Token::Kind TokenStreamIterator::LookaheadTokenKind(intptr_t num_tokens) const {
-  return tokens_.KindAt(token_position_ + num_tokens);
-}
-
-
-RawObject* TokenStreamIterator::CurrentToken() const {
-  return tokens_.TokenAt(token_position_);
-}
-
-
-RawString* TokenStreamIterator::CurrentLiteral() const {
-  return tokens_.LiteralAt(token_position_);
-}
-
-
 struct Parser::Block : public ZoneAllocated {
   Block(Block* outer_block, LocalScope* local_scope, SequenceNode* seq)
     : parent(outer_block), scope(local_scope), statements(seq) {
     ASSERT(scope != NULL);
     ASSERT(statements != NULL);
   }
   Block* parent;  // Enclosing block, or NULL if outermost.
   LocalScope* scope;
   SequenceNode* statements;
 };
@@ skipping 115 matching lines @@
   token_kind_ = Token::kILLEGAL;
 }
 
 
 void Parser::ParseCompilationUnit(const Library& library,
                                   const Script& script) {
   ASSERT(Isolate::Current()->long_jump_base()->IsSafeToJump());
   TimerScope timer(FLAG_compiler_stats, &CompilerStats::parser_timer);
   Parser parser(script, library);
   parser.ParseTopLevel();
-  if (FLAG_compiler_stats) {
-    CompilerStats::num_tokens_total += parser.tokens_iterator_.NumberOfTokens();
-  }
 }
 
 
 Token::Kind Parser::CurrentToken() {
   if (token_kind_ == Token::kILLEGAL) {
     token_kind_ = tokens_iterator_.CurrentTokenKind();
     if (token_kind_ == Token::kERROR) {
       ErrorMsg(TokenPos(), CurrentLiteral()->ToCString());
     }
   }
@@ skipping 8261 matching lines @@
 void Parser::SkipQualIdent() {
   ASSERT(IsIdentifier());
   ConsumeToken();
   if (CurrentToken() == Token::kPERIOD) {
     ConsumeToken();  // Consume the kPERIOD token.
     ExpectIdentifier("identifier expected after '.'");
   }
 }
 
 }  // namespace dart