Changes to shrink the token stream representation from two words to one word.

vm/object.h

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 #ifndef VM_OBJECT_H_
 #define VM_OBJECT_H_
 
 #include "include/dart_api.h"
 #include "platform/assert.h"
 #include "platform/utils.h"
@@ skipping 126 matching lines @@
     kVoidClass,
     kUnresolvedClassClass,
     kTypeClass,
     kTypeParameterClass,
     kInstantiatedTypeClass,
     kAbstractTypeArgumentsClass,
     kTypeArgumentsClass,
     kInstantiatedTypeArgumentsClass,
     kFunctionClass,
     kFieldClass,
+    kLiteralTokenClass,
     kTokenStreamClass,
     kScriptClass,
     kLibraryClass,
     kLibraryPrefixClass,
     kCodeClass,
     kInstructionsClass,
     kPcDescriptorsClass,
     kLocalVarDescriptorsClass,
     kExceptionHandlersClass,
     kContextClass,
@@ skipping 115 matching lines @@
   }
   static RawClass* abstract_type_arguments_class() {
     return abstract_type_arguments_class_;
   }
   static RawClass* type_arguments_class() { return type_arguments_class_; }
   static RawClass* instantiated_type_arguments_class() {
       return instantiated_type_arguments_class_;
   }
   static RawClass* function_class() { return function_class_; }
   static RawClass* field_class() { return field_class_; }
+  static RawClass* literal_token_class() { return literal_token_class_; }
   static RawClass* token_stream_class() { return token_stream_class_; }
   static RawClass* script_class() { return script_class_; }
   static RawClass* library_class() { return library_class_; }
   static RawClass* library_prefix_class() { return library_prefix_class_; }
   static RawClass* code_class() { return code_class_; }
   static RawClass* instructions_class() { return instructions_class_; }
   static RawClass* pc_descriptors_class() { return pc_descriptors_class_; }
   static RawClass* var_descriptors_class() { return var_descriptors_class_; }
   static RawClass* exception_handlers_class() {
     return exception_handlers_class_;
@@ skipping 91 matching lines @@
   static RawClass* type_class_;  // Class of Type.
   static RawClass* type_parameter_class_;  // Class of TypeParameter vm object.
   static RawClass* instantiated_type_class_;  // Class of InstantiatedType.
   // Class of AbstractTypeArguments vm object.
   static RawClass* abstract_type_arguments_class_;
   // Class of the TypeArguments vm object.
   static RawClass* type_arguments_class_;
   static RawClass* instantiated_type_arguments_class_;  // Class of Inst..ments.
   static RawClass* function_class_;  // Class of the Function vm object.
   static RawClass* field_class_;  // Class of the Field vm object.
+  static RawClass* literal_token_class_;  // Class of LiteralToken vm object.
   static RawClass* token_stream_class_;  // Class of the TokenStream vm object.
   static RawClass* script_class_;  // Class of the Script vm object.
   static RawClass* library_class_;  // Class of the Library vm object.
   static RawClass* library_prefix_class_;  // Class of Library prefix vm object.
   static RawClass* code_class_;  // Class of the Code vm object.
   static RawClass* instructions_class_;  // Class of the Instructions vm object.
   static RawClass* pc_descriptors_class_;  // Class of PcDescriptors vm object.
   static RawClass* var_descriptors_class_;  // Class of LocalVarDescriptors.
   static RawClass* exception_handlers_class_;  // Class of ExceptionHandlers.
   static RawClass* context_class_;  // Class of the Context vm object.
@@ skipping 1141 matching lines @@
   void set_token_index(intptr_t token_index) const {
     raw_ptr()->token_index_ = token_index;
   }
   static RawField* New();
 
   HEAP_OBJECT_IMPLEMENTATION(Field, Object);
   friend class Class;
 };
 
 
+class LiteralToken : public Object {
+ public:
+  Token::Kind kind() const { return raw_ptr()->kind_; }
+  RawString* literal() const { return raw_ptr()->literal_; }
+  RawObject* value() const { return raw_ptr()->value_; }
+
+  RawLiteralToken* Canonicalize() const;
+
+  static intptr_t InstanceSize() {
+    return RoundedAllocationSize(sizeof(RawLiteralToken));
+  }
+
+  static RawLiteralToken* New();
+  static RawLiteralToken* New(Token::Kind kind, const String& literal);
+  static RawLiteralToken* NewCanonical(Token::Kind kind,
+                                       const String& literal);
+
+ private:
+  void set_kind(Token::Kind kind) const { raw_ptr()->kind_ = kind; }
+  void set_literal(const String& literal) const;
+  void set_value(const Object& value) const;
+
+  HEAP_OBJECT_IMPLEMENTATION(LiteralToken, Object);
+  friend class Class;
+};
+
+
 class TokenStream : public Object {
  public:
   inline intptr_t Length() const;
 
   inline Token::Kind KindAt(intptr_t index) const;
 
   void SetTokenAt(intptr_t index, Token::Kind kind, const String& literal);
+  void SetTokenAt(intptr_t index, const Object& token);
 
-  RawObject* LiteralAt(intptr_t index) const {
-    return *EntryAddr(index, RawTokenStream::kLiteralEntry);
-  }
+  RawObject* TokenAt(intptr_t index) const;
+  RawString* LiteralAt(intptr_t index) const;
 
   static intptr_t InstanceSize() {
     ASSERT(sizeof(RawTokenStream) == OFFSET_OF(RawTokenStream, data_));
     return 0;
   }
   static intptr_t InstanceSize(intptr_t len) {
     return RoundedAllocationSize(
         sizeof(RawTokenStream) +
             (len * RawTokenStream::kNumberOfEntries * kWordSize));
   }
   static intptr_t StreamLength(intptr_t len) {
     return (len * RawTokenStream::kNumberOfEntries);
   }
 
   static RawTokenStream* New(intptr_t length);
   static RawTokenStream* New(const Scanner::GrowableTokenStream& tokens);
 
  private:
   void SetLength(intptr_t value) const;
 
   RawObject** EntryAddr(intptr_t index, intptr_t entry_offset) const {
     ASSERT((index >=0) && (index < Length()));
     intptr_t data_index =
         (index * RawTokenStream::kNumberOfEntries) + entry_offset;

[hausner, 2012/02/25 00:41:57]

Does it still make sense to perform this computation when we now only have one
entry per token, i.e. kNumberOfEntries == 1? I think this can be eliminated.

[siva, 2012/02/28 19:57:31]

I added an ASSERT(kNumberofEntries == 1);
and eliminated the computation.

On 2012/02/25 00:41:57, hausner wrote:

     return &raw_ptr()->data_[data_index];
   }
 
   RawSmi** SmiAddr(intptr_t index, intptr_t entry_offset) const {
     return reinterpret_cast<RawSmi**>(EntryAddr(index, entry_offset));
   }
 
   HEAP_OBJECT_IMPLEMENTATION(TokenStream, Object);
   friend class Class;
 };
@@ skipping 2127 matching lines @@
   raw_ptr()->value_ = Smi::New(value);
 }
 
 
 intptr_t TokenStream::Length() const {
   return Smi::Value(raw_ptr()->length_);
 }
 
 
 Token::Kind TokenStream::KindAt(intptr_t index) const {
-  return static_cast<Token::Kind>(
-      Smi::Value(*SmiAddr(index, RawTokenStream::kKindEntry)));
+  const Object& obj = Object::Handle(TokenAt(index));
+  if (obj.IsSmi()) {
+    return static_cast<Token::Kind>(
+        Smi::Value(reinterpret_cast<RawSmi*>(obj.raw())));
+  } else if (obj.IsLiteralToken()) {
+    LiteralToken& token = LiteralToken::Handle();
+    token ^= obj.raw();
+    return token.kind();
+  }
+  ASSERT(obj.IsString());  // Must be an identifier.
+  return Token::kIDENT;
 }
 
 
 void Context::SetAt(intptr_t index, const Instance& value) const {
   StorePointer(InstanceAddr(index), value.raw());
 }
 
 }  // namespace dart
 
 #endif  // VM_OBJECT_H_