Represent tokens as a compressed stream instead of an array.

vm/object.h

Index: vm/object.h
===================================================================
--- vm/object.h	(revision 9303)
+++ vm/object.h	(working copy)
@@ -10,6 +10,7 @@
 #include "platform/utils.h"
 #include "vm/bitmap.h"
 #include "vm/dart.h"
+#include "vm/datastream.h"
 #include "vm/globals.h"
 #include "vm/handles.h"
 #include "vm/heap.h"
@@ -1684,21 +1685,18 @@
  public:
   inline intptr_t Length() const;
 
-  inline Token::Kind KindAt(intptr_t index) const;
+  RawArray* TokenObjects() const;
+  void SetTokenObjects(const Array& value) const;
 
-  void SetTokenAt(intptr_t index, Token::Kind kind, const String& literal);
-  void SetTokenAt(intptr_t index, const Object& token);
-
-  RawObject* TokenAt(intptr_t index) const;
-  RawString* LiteralAt(intptr_t index) const;
   RawString* GenerateSource() const;
+  intptr_t ComputeSourcePosition(intptr_t tok_pos) const;
 
   static intptr_t InstanceSize() {
     ASSERT(sizeof(RawTokenStream) == OFFSET_OF(RawTokenStream, data_));
     return 0;
   }
   static intptr_t InstanceSize(intptr_t len) {
-    return RoundedAllocationSize(sizeof(RawTokenStream) + (len * kWordSize));
+    return RoundedAllocationSize(sizeof(RawTokenStream) + len);
   }
   static intptr_t StreamLength(intptr_t len) {
     return len;
@@ -1710,20 +1708,59 @@
  private:
   void SetLength(intptr_t value) const;
 
-  RawObject** EntryAddr(intptr_t index) const {
-    ASSERT((index >=0) && (index < Length()));
-    return &raw_ptr()->data_[index];
+  uint8_t* EntryAddr(intptr_t token_pos) const {
+    ASSERT((token_pos >=0) && (token_pos < Length()));
+    return &raw_ptr()->data_[token_pos];
   }
 
-  RawSmi** SmiAddr(intptr_t index) const {
-    return reinterpret_cast<RawSmi**>(EntryAddr(index));
-  }
-
   HEAP_OBJECT_IMPLEMENTATION(TokenStream, Object);
   friend class Class;
+  friend class TokenStreamIterator;
 };
 
 
+// The class TokenStreamIterator encapsulates iteration over the tokens
+// in a TokenStream object.
+class TokenStreamIterator : ValueObject {
+ public:
+  TokenStreamIterator(const TokenStream& tokens, intptr_t token_pos);
+
+  bool IsValid() const;
+
+  inline Token::Kind CurrentTokenKind() const {
+    return cur_token_kind_;
+  }
+
+  Token::Kind LookaheadTokenKind(intptr_t num_tokens);
+
+  intptr_t CurrentPosition() const;
+  void SetCurrentPosition(intptr_t value);
+
+  void Advance();
+
+  RawObject* CurrentToken() const;
+  RawString* CurrentLiteral() const;
+  RawString* MakeLiteralToken(const Object& obj) const;
+
+ private:
+  // Read token from the token stream (could be a simple token or an index
+  // into the token objects array for IDENT or literal tokens).
+  intptr_t ReadToken() {
+    int64_t value = stream_.ReadUnsigned();
+    ASSERT((value >= 0) && (value <= kIntptrMax));
+    return value;
+  }
+
+  const TokenStream& tokens_;
+  ReadStream stream_;
+  Array& token_objects_;
+  Object& obj_;
+  intptr_t cur_token_pos_;
+  Token::Kind cur_token_kind_;
+  intptr_t cur_token_obj_index_;
+};
+
+
 class Script : public Object {
  public:
   RawString* url() const { return raw_ptr()->url_; }
@@ -5067,25 +5104,10 @@
 
 
 intptr_t TokenStream::Length() const {
-  return Smi::Value(raw_ptr()->length_);
+  return raw_ptr()->length_;
 }
 
 
-Token::Kind TokenStream::KindAt(intptr_t index) const {
-  const Object& obj = Object::Handle(TokenAt(index));
-  if (obj.IsSmi()) {
-    return static_cast<Token::Kind>(
-        Smi::Value(reinterpret_cast<RawSmi*>(obj.raw())));
-  } else if (obj.IsLiteralToken()) {
-    LiteralToken& token = LiteralToken::Handle();
-    token ^= obj.raw();
-    return token.kind();
-  }
-  ASSERT(obj.IsString());  // Must be an identifier.
-  return Token::kIDENT;
-}
-
-
 void Context::SetAt(intptr_t index, const Instance& value) const {
   StorePointer(InstanceAddr(index), value.raw());
 }