Changes to shrink the token stream representation from two words to one word.

vm/raw_object.h

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 #ifndef VM_RAW_OBJECT_H_
 #define VM_RAW_OBJECT_H_
 
 #include "platform/assert.h"
 #include "vm/globals.h"
+#include "vm/token.h"
 #include "vm/snapshot.h"
 
 #include "include/dart_api.h"
 
 namespace dart {
 
 // Macrobatics to define the Object hierarchy of VM implementation classes.
 #define CLASS_LIST_NO_OBJECT(V)                                                \
   V(Class)                                                                     \
   V(UnresolvedClass)                                                           \
   V(AbstractType)                                                              \
     V(Type)                                                                    \
     V(TypeParameter)                                                           \
     V(InstantiatedType)                                                        \
   V(AbstractTypeArguments)                                                     \
     V(TypeArguments)                                                           \
     V(InstantiatedTypeArguments)                                               \
   V(Function)                                                                  \
   V(Field)                                                                     \
+  V(LiteralToken)                                                              \
   V(TokenStream)                                                               \
   V(Script)                                                                    \
   V(Library)                                                                   \
   V(LibraryPrefix)                                                             \
   V(Code)                                                                      \
   V(Instructions)                                                              \
   V(PcDescriptors)                                                             \
   V(LocalVarDescriptors)                                                       \
   V(ExceptionHandlers)                                                         \
   V(Context)                                                                   \
@@ skipping 481 matching lines @@
   RawInstance* value_;  // Offset for instance and value for static fields.
   RawObject** to() { return reinterpret_cast<RawObject**>(&ptr()->value_); }
 
   intptr_t token_index_;
   bool is_static_;
   bool is_final_;
   bool has_initializer_;
 };
 
 
+class RawLiteralToken : public RawObject {
+  RAW_HEAP_OBJECT_IMPLEMENTATION(LiteralToken);
+
+  RawObject** from() {
+    return reinterpret_cast<RawObject**>(&ptr()->literal_);
+  }
+  RawString* literal_;  // The actual token string.

[hausner, 2012/02/25 00:41:57]

// literal characters as they appear in source text.

[siva, 2012/02/28 19:57:31]

Done.

+  RawObject* value_;  // The actual object corresponding to the token.
+  RawObject** to() {
+    return reinterpret_cast<RawObject**>(&ptr()->value_);
+  }
+  Token::Kind kind_;  // The literal kind (string, integer, double).
+
+  friend class SnapshotReader;
+};
+
+
 class RawTokenStream : public RawObject {
   RAW_HEAP_OBJECT_IMPLEMENTATION(TokenStream);
 
   enum {
-    kKindEntry = 0,
-    kLiteralEntry,
+    kTokenEntry = 0,
     kNumberOfEntries
   };
 
   RawObject** from() {
     return reinterpret_cast<RawObject**>(&ptr()->private_key_);
   }
   RawString* private_key_;  // Key used for private identifiers.
   RawSmi* length_;  // Number of tokens.
 
   // Variable length data follows here.
@@ skipping 552 matching lines @@
   RawString* target_name_;  // Name of target function.
   RawArray* ic_data_;  // Contains test classes and target function.
   intptr_t id_;  // Parser node id corresponding to this IC.
   intptr_t num_args_tested_;  // Number of arguments tested in IC.
 };
 
 
 }  // namespace dart
 
 #endif  // VM_RAW_OBJECT_H_