Changes to shrink the token stream representation from two words to one word.

vm/scanner.cc

Index: vm/scanner.cc
===================================================================
--- vm/scanner.cc	(revision 4719)
+++ vm/scanner.cc	(working copy)
@@ -7,6 +7,7 @@
 #include "platform/assert.h"
 #include "vm/flags.h"
 #include "vm/object.h"
+#include "vm/object_store.h"
 #include "vm/thread.h"
 #include "vm/token.h"
 
@@ -15,6 +16,13 @@
 DEFINE_FLAG(bool, print_tokens, false, "Print scanned tokens.");
 
 void Scanner::InitKeywordTable() {
+  ObjectStore* object_store = Isolate::Current()->object_store();
+  keyword_symbol_table_ = object_store->keyword_symbols();
+  if (keyword_symbol_table_.IsNull()) {
+    object_store->InitKeywordTable();
+    keyword_symbol_table_ = object_store->keyword_symbols();
+    ASSERT(!keyword_symbol_table_.IsNull());
+  }
   for (int i = 0; i < Token::numKeywords; i++) {
     Token::Kind token = static_cast<Token::Kind>(Token::kFirstKeyword + i);
     keywords_[i].kind = token;
@@ -48,7 +56,8 @@
     : source_(src),
       source_length_(src.Length()),
       saved_context_(NULL),
-      private_key_(String::ZoneHandle(private_key.raw())) {
+      private_key_(String::ZoneHandle(private_key.raw())),
+      keyword_symbol_table_(Array::ZoneHandle()) {
   Reset();
   InitKeywordTable();
 }
@@ -261,8 +270,13 @@
       }
       if (char_pos == ident_length) {
         if (keywords_[i].keyword_symbol == NULL) {
-          keywords_[i].keyword_symbol = &String::ZoneHandle(
-              String::NewSymbol(source_, ident_pos, ident_length));
+          String& symbol = String::ZoneHandle();
+          symbol ^= keyword_symbol_table_.At(i);
+          if (symbol.IsNull()) {
+            symbol = String::NewSymbol(source_, ident_pos, ident_length);
+            keyword_symbol_table_.SetAt(i, symbol);
+          }
+          keywords_[i].keyword_symbol = &symbol;
         }
         current_token_.literal = keywords_[i].keyword_symbol;
         current_token_.kind = keywords_[i].kind;