Changes to shrink the token stream representation from two words to one word.

vm/scanner.cc

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 #include "vm/scanner.h"
 
 #include "platform/assert.h"
 #include "vm/flags.h"
 #include "vm/object.h"
+#include "vm/object_store.h"
 #include "vm/thread.h"
 #include "vm/token.h"
 
 namespace dart {
 
 DEFINE_FLAG(bool, print_tokens, false, "Print scanned tokens.");
 
 void Scanner::InitKeywordTable() {
+  ObjectStore* object_store = Isolate::Current()->object_store();
+  keyword_symbol_table_ = object_store->keyword_symbols();
+  if (keyword_symbol_table_.IsNull()) {
+    object_store->InitKeywordTable();
+    keyword_symbol_table_ = object_store->keyword_symbols();
+    ASSERT(!keyword_symbol_table_.IsNull());
+  }
   for (int i = 0; i < Token::numKeywords; i++) {
     Token::Kind token = static_cast<Token::Kind>(Token::kFirstKeyword + i);
     keywords_[i].kind = token;
     keywords_[i].keyword_chars = Token::Str(token);
     keywords_[i].keyword_len = strlen(Token::Str(token));
     keywords_[i].keyword_symbol = NULL;
   }
 }
 
 
@@ skipping 13 matching lines @@
   c0_pos_.line = 1;
   c0_pos_.column = 0;
   ReadChar();
 }
 
 
 Scanner::Scanner(const String& src, const String& private_key)
     : source_(src),
       source_length_(src.Length()),
       saved_context_(NULL),
-      private_key_(String::ZoneHandle(private_key.raw())) {
+      private_key_(String::ZoneHandle(private_key.raw())),
+      keyword_symbol_table_(Array::ZoneHandle()) {
   Reset();
   InitKeywordTable();
 }
 
 Scanner::~Scanner() {
   while (saved_context_ != NULL) {
     ScanContext* ctx = saved_context_;
     saved_context_ = ctx->next;
     delete ctx;
   }
@@ skipping 192 matching lines @@
          keywords_[i].keyword_chars[0] <= ident_char0) {
     if (keywords_[i].keyword_len == ident_length) {
       const char* keyword = keywords_[i].keyword_chars;
       int char_pos = 0;
       while ((char_pos < ident_length) &&
              (keyword[char_pos] == source_.CharAt(ident_pos + char_pos))) {
         char_pos++;
       }
       if (char_pos == ident_length) {
         if (keywords_[i].keyword_symbol == NULL) {
-          keywords_[i].keyword_symbol = &String::ZoneHandle(
-              String::NewSymbol(source_, ident_pos, ident_length));
+          String& symbol = String::ZoneHandle();
+          symbol ^= keyword_symbol_table_.At(i);
+          if (symbol.IsNull()) {
+            symbol = String::NewSymbol(source_, ident_pos, ident_length);
+            keyword_symbol_table_.SetAt(i, symbol);
+          }
+          keywords_[i].keyword_symbol = &symbol;
         }
         current_token_.literal = keywords_[i].keyword_symbol;
         current_token_.kind = keywords_[i].kind;
         return;
       }
     }
     i++;
   }
 
   // We did not read a keyword.
@@ skipping 621 matching lines @@
               "%c%"PRIxPTR, kPrivateKeySeparator, key_value);
   const String& result = String::Handle(String::New(private_key));
   return result.raw();
 }
 
 
 void Scanner::InitOnce() {
 }
 
 }  // namespace dart