Changes to shrink the token stream representation from two words to one word.

vm/token.h

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 #ifndef VM_TOKEN_H_
 #define VM_TOKEN_H_
 
 #include "platform/assert.h"
 
 namespace dart {
@@ skipping 200 matching lines @@
   }
 
   static bool IsInstanceofOperator(Kind tok) {
     return (tok == kIS) || (tok == kISNOT);
   }
 
   static bool IsPseudoKeyword(Kind tok) {
     return (Attributes(tok) & kPseudoKeyword) != 0;
   }
 
+  static bool IsKeyword(Kind tok) {
+    return (Attributes(tok) & kKeyword) != 0;
+  }
+
   static bool IsIdentifier(Kind tok) {
     return (tok == kIDENT) || IsPseudoKeyword(tok);
   }
 
   static const char* Name(Kind tok) {
     ASSERT(tok < kNumTokens);
     return name_[tok];
   }
 
   static const char* Str(Kind tok) {
@@ skipping 15 matching lines @@
     ASSERT(tok < kNumTokens);
     return IsRelationalOperator(tok) ||
            (tok == kEQ) ||
            (tok >= kADD && tok <= kMOD) ||  // Arithmetic operations.
            (tok >= kBIT_OR && tok <= kSHR) ||  // Bit operations.
            (tok == kINDEX) ||
            (tok == kASSIGN_INDEX) ||
            (tok == kNEGATE);
   }
 
+  static bool NeedsLiteralToken(Kind tok) {
+    ASSERT(tok < kNumTokens);
+    return ((tok == Token::kINTEGER) ||
+            (tok == Token::kSTRING) ||
+            (tok == Token::kINTERPOL_VAR) ||
+            (tok == Token::kERROR) ||
+            (tok == Token::kDOUBLE));
+  }
+
  private:
   static const char* name_[kNumTokens];
   static const char* tok_str_[kNumTokens];
   static const uint8_t precedence_[kNumTokens];
   static const Attribute attributes_[kNumTokens];
 };
 
 
 }  // namespace dart
 
 #endif  // VM_TOKEN_H_