Represent tokens as a compressed stream instead of an array.

vm/object.h

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 #ifndef VM_OBJECT_H_
 #define VM_OBJECT_H_
 
 #include "include/dart_api.h"
 #include "platform/assert.h"
 #include "platform/utils.h"
 #include "vm/bitmap.h"
 #include "vm/dart.h"
+#include "vm/datastream.h"
 #include "vm/globals.h"
 #include "vm/handles.h"
 #include "vm/heap.h"
 #include "vm/isolate.h"
 #include "vm/os.h"
 #include "vm/raw_object.h"
 #include "vm/scanner.h"
 
 namespace dart {
 
@@ skipping 1654 matching lines @@
 
   HEAP_OBJECT_IMPLEMENTATION(LiteralToken, Object);
   friend class Class;
 };
 
 
 class TokenStream : public Object {
  public:
   inline intptr_t Length() const;
 
-  inline Token::Kind KindAt(intptr_t index) const;
+  RawArray* TokenObjects() const;
+  void SetTokenObjects(const Array& value) const;
 
-  void SetTokenAt(intptr_t index, Token::Kind kind, const String& literal);
-  void SetTokenAt(intptr_t index, const Object& token);
-
-  RawObject* TokenAt(intptr_t index) const;
-  RawString* LiteralAt(intptr_t index) const;
   RawString* GenerateSource() const;
+  intptr_t ComputeSourcePosition(intptr_t tok_pos) const;
 
   static intptr_t InstanceSize() {
     ASSERT(sizeof(RawTokenStream) == OFFSET_OF(RawTokenStream, data_));
     return 0;
   }
   static intptr_t InstanceSize(intptr_t len) {
-    return RoundedAllocationSize(sizeof(RawTokenStream) + (len * kWordSize));
+    return RoundedAllocationSize(sizeof(RawTokenStream) + len);
   }
   static intptr_t StreamLength(intptr_t len) {
     return len;
   }
 
   static RawTokenStream* New(intptr_t length);
   static RawTokenStream* New(const Scanner::GrowableTokenStream& tokens);
 
  private:
   void SetLength(intptr_t value) const;
 
-  RawObject** EntryAddr(intptr_t index) const {
-    ASSERT((index >=0) && (index < Length()));
-    return &raw_ptr()->data_[index];
-  }
-
-  RawSmi** SmiAddr(intptr_t index) const {
-    return reinterpret_cast<RawSmi**>(EntryAddr(index));
+  uint8_t* EntryAddr(intptr_t token_pos) const {
+    ASSERT((token_pos >=0) && (token_pos < Length()));
+    return &raw_ptr()->data_[token_pos];
   }
 
   HEAP_OBJECT_IMPLEMENTATION(TokenStream, Object);
   friend class Class;
+  friend class TokenStreamIterator;
+};
+
+
+// The class TokenStreamIterator encapsulates iteration over the tokens
+// in a TokenStream object.
+class TokenStreamIterator : ValueObject {
+ public:
+  TokenStreamIterator(const TokenStream& tokens, intptr_t token_pos);
+
+  bool IsValid() const;
+
+  inline Token::Kind CurrentTokenKind() const {
+    return cur_token_kind_;
+  }
+
+  Token::Kind LookaheadTokenKind(intptr_t num_tokens);
+
+  intptr_t CurrentPosition() const;
+  void SetCurrentPosition(intptr_t value);
+
+  void Advance();
+
+  RawObject* CurrentToken() const;
+  RawString* CurrentLiteral() const;
+  RawString* MakeLiteralToken(const Object& obj) const;
+
+ private:
+  // Read token from the token stream (could be a simple token or an index
+  // into the token objects array for IDENT or literal tokens).
+  intptr_t ReadToken() {
+    int64_t value = stream_.ReadUnsigned();
+    ASSERT((value >= 0) && (value <= kIntptrMax));
+    return value;
+  }
+
+  const TokenStream& tokens_;
+  ReadStream stream_;
+  Array& token_objects_;
+  Object& obj_;
+  intptr_t cur_token_pos_;
+  Token::Kind cur_token_kind_;
+  intptr_t cur_token_obj_index_;
 };
 
 
 class Script : public Object {
  public:
   RawString* url() const { return raw_ptr()->url_; }
   RawString* source() const;
   RawScript::Kind kind() const { return raw_ptr()->kind_; }
 
   RawTokenStream* tokens() const { return raw_ptr()->tokens_; }
@@ skipping 3326 matching lines @@
 }
 
 
 void Field::SetOffset(intptr_t value) const {
   ASSERT(!is_static());  // SetOffset is valid only for instance fields.
   raw_ptr()->value_ = Smi::New(value);
 }
 
 
 intptr_t TokenStream::Length() const {
-  return Smi::Value(raw_ptr()->length_);
-}
-
-
-Token::Kind TokenStream::KindAt(intptr_t index) const {
-  const Object& obj = Object::Handle(TokenAt(index));
-  if (obj.IsSmi()) {
-    return static_cast<Token::Kind>(
-        Smi::Value(reinterpret_cast<RawSmi*>(obj.raw())));
-  } else if (obj.IsLiteralToken()) {
-    LiteralToken& token = LiteralToken::Handle();
-    token ^= obj.raw();
-    return token.kind();
-  }
-  ASSERT(obj.IsString());  // Must be an identifier.
-  return Token::kIDENT;
+  return raw_ptr()->length_;
 }
 
 
 void Context::SetAt(intptr_t index, const Instance& value) const {
   StorePointer(InstanceAddr(index), value.raw());
 }
 
 
 intptr_t Stackmap::SizeInBits() const {
   return (Smi::Value(raw_ptr()->bitmap_size_in_bytes_) * kBitsPerByte);
 }
 
 }  // namespace dart
 
 #endif  // VM_OBJECT_H_