Eliminate GrowableTokenStream

runtime/vm/scanner.cc

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 #include "vm/scanner.h"
 
 #include "platform/assert.h"
 #include "vm/dart.h"
 #include "vm/flags.h"
 #include "vm/object.h"
 #include "vm/object_store.h"
 #include "vm/symbols.h"
 #include "vm/token.h"
 #include "vm/unicode.h"
 
 namespace dart {
 
-DEFINE_FLAG(bool, print_tokens, false, "Print scanned tokens.");
-
-
 // Quick access to the locally defined zone() and thread() methods.
 #define Z (zone())
 #define T (thread())
 
 
 class ScanContext : public ZoneAllocated {
  public:
   explicit ScanContext(Scanner* scanner)
       :  next_(scanner->saved_context_),
          string_delimiter_(scanner->string_delimiter_),
@@ skipping 143 matching lines @@
   }
   for (int i =  1; i < str.Length(); i++) {
     if (!IsIdentChar(CallCharAt()(str, i))) {
       return false;
     }
   }
   return true;
 }
 
 
-// This method is used when parsing integers and doubles in Dart code. We
+// This method is used when parsing integers in Dart code. We
 // are reusing the Scanner's handling of number literals in that situation.
-bool Scanner::IsValidLiteral(const Scanner::GrowableTokenStream& tokens,
-                             Token::Kind literal_kind,
+bool Scanner::IsValidInteger(const String& str,
                              bool* is_positive,
                              const String** value) {
-  if ((tokens.length() == 2) &&
-      (tokens[0].kind == literal_kind) &&
+  Scanner s(str, Symbols::Empty());
+  TokenDescriptor tokens[3];
+  s.Scan();
+  tokens[0] = s.current_token();
+  s.Scan();
+  tokens[1] = s.current_token();
+  s.Scan();
+  tokens[2] = s.current_token();
+
+  if ((tokens[0].kind == Token::kINTEGER) &&
       (tokens[1].kind == Token::kEOS)) {
     *is_positive = true;
     *value = tokens[0].literal;
     return true;
   }
-  if ((tokens.length() == 3) &&
-      ((tokens[0].kind == Token::kADD) ||
+  if (((tokens[0].kind == Token::kADD) ||
        (tokens[0].kind == Token::kSUB)) &&
-      (tokens[1].kind == literal_kind) &&
+      (tokens[1].kind == Token::kINTEGER) &&
       (tokens[2].kind == Token::kEOS)) {
     // Check there is no space between "+/-" and number.
     if ((tokens[0].offset + 1) != tokens[1].offset) {
       return false;
     }
     *is_positive = tokens[0].kind == Token::kADD;
     *value = tokens[1].literal;
     return true;
   }
   return false;
@@ skipping 668 matching lines @@
           OS::SNPrint(msg, sizeof(msg),
                       "unexpected character: '%s' (U+%04X)\n", utf8_char, c0_);
           ErrorMsg(msg);
           ReadChar();
         }
     }
   } while (current_token_.kind == Token::kWHITESP);
 }
 
 
-void Scanner::ScanAll(GrowableTokenStream* token_stream) {
+void Scanner::ScanAll(TokenCollector* collector) {
   Reset();
   do {
     Scan();
-
     bool inserted_new_lines = false;
     for (intptr_t diff = current_token_.position.line - prev_token_line_;
          diff > 0;
          diff--) {
       newline_token_.position.line = current_token_.position.line - diff;
-      token_stream->Add(newline_token_);
+      collector->AddToken(newline_token_);
       inserted_new_lines = true;
     }
-
     if (inserted_new_lines &&
         ((current_token_.kind == Token::kINTERPOL_VAR) ||
          (current_token_.kind == Token::kINTERPOL_START))) {
-      // NOTE: If this changes, be sure to update
-      // Script::GenerateLineNumberArray to stay in sync.
-      empty_string_token_.position.line = current_token_.position.line;
-      token_stream->Add(empty_string_token_);
-    }
-    token_stream->Add(current_token_);
+          // NOTE: If this changes, be sure to update
+          // Script::GenerateLineNumberArray to stay in sync.
+          empty_string_token_.position.line = current_token_.position.line;
+          collector->AddToken(empty_string_token_);
+        }
+    collector->AddToken(current_token_);
     prev_token_line_ = current_token_.position.line;
   } while (current_token_.kind != Token::kEOS);
 }
 
 
 void Scanner::ScanTo(TokenPosition token_index) {
   TokenPosition index = TokenPosition::kMinSource;
   Reset();
   do {
     Scan();
-
     bool inserted_new_lines = false;
     for (intptr_t diff = current_token_.position.line - prev_token_line_;
          diff > 0;
          diff--) {
       // Advance the index to account for tokens added in ScanAll.
       index.Next();
       inserted_new_lines = true;
     }
-
     if (inserted_new_lines &&
         ((current_token_.kind == Token::kINTERPOL_VAR) ||
          (current_token_.kind == Token::kINTERPOL_START))) {
           // Advance the index to account for tokens added in ScanAll.
           index.Next();
     }
     index.Next();
     prev_token_line_ = current_token_.position.line;
   } while ((token_index >= index) && (current_token_.kind != Token::kEOS));
 }
 
 
-const Scanner::GrowableTokenStream& Scanner::GetStream() {
-  GrowableTokenStream* ts = new(Z) GrowableTokenStream(128);
-  ScanAll(ts);
-  if (FLAG_print_tokens) {
-    Scanner::PrintTokens(*ts);
-  }
-  return *ts;
-}
-
-
-void Scanner::PrintTokens(const GrowableTokenStream& ts) {
-  int currentLine = -1;
-  for (int i = 0; i < ts.length(); i++) {
-    const TokenDescriptor& td = ts[i];
-    if (currentLine != td.position.line) {
-      currentLine = td.position.line;
-      OS::Print("\n%d (%d): ", currentLine, i);
-    }
-    OS::Print("%s ", Token::Name(td.kind));
-  }
-  OS::Print("\n");
-}
-
-
 void Scanner::InitOnce() {
   ASSERT(Isolate::Current() == Dart::vm_isolate());
   for (int i = 0; i < kNumLowercaseChars; i++) {
     keywords_char_offset_[i] = Token::kNumKeywords;
   }
   for (int i = 0; i < Token::kNumKeywords; i++) {
     Token::Kind token = static_cast<Token::Kind>(Token::kFirstKeyword + i);
     keywords_[i].kind = token;
     keywords_[i].keyword_chars = Token::Str(token);
     keywords_[i].keyword_len = strlen(Token::Str(token));
     keywords_[i].keyword_symbol = &Symbols::Token(token);
 
     int ch = keywords_[i].keyword_chars[0] - 'a';
     if (keywords_char_offset_[ch] == Token::kNumKeywords) {
       keywords_char_offset_[ch] = i;
     }
   }
 }
 
 }  // namespace dart