Use external byte arrays for token stream, this moves the token stream out of the isolate heap. For…

vm/object.h

Index: vm/object.h
===================================================================
--- vm/object.h	(revision 11973)
+++ vm/object.h	(working copy)
@@ -1822,11 +1822,12 @@
 
 class TokenStream : public Object {
  public:
-  inline intptr_t Length() const;
-
   RawArray* TokenObjects() const;
   void SetTokenObjects(const Array& value) const;
 
+  RawExternalUint8Array* GetStream() const;
+  void SetStream(const ExternalUint8Array& stream) const;
+
   RawString* GenerateSource() const;
   intptr_t ComputeSourcePosition(intptr_t tok_pos) const;
   intptr_t ComputeTokenPosition(intptr_t src_pos) const;
@@ -1835,14 +1836,8 @@
   static const intptr_t kMaxElements = kSmiMax / kBytesPerElement;
 
   static intptr_t InstanceSize() {
-    ASSERT(sizeof(RawTokenStream) == OFFSET_OF(RawTokenStream, data_));
-    return 0;
+    return RoundedAllocationSize(sizeof(RawTokenStream));
   }
-  static intptr_t InstanceSize(intptr_t len) {
-    ASSERT(0 <= len && len <= kMaxElements);
-    return RoundedAllocationSize(
-        sizeof(RawTokenStream) + (len * kBytesPerElement));
-  }
 
   static RawTokenStream* New(intptr_t length);
   static RawTokenStream* New(const Scanner::GrowableTokenStream& tokens,
@@ -1874,28 +1869,28 @@
    private:
     // Read token from the token stream (could be a simple token or an index
     // into the token objects array for IDENT or literal tokens).
-    intptr_t ReadToken();
-    uint8_t ReadByte();
+    intptr_t ReadToken() {
+      int64_t value = stream_.ReadUnsigned();
+      ASSERT((value >= 0) && (value <= kIntptrMax));
+      return value;
+    }
 
     const TokenStream& tokens_;
+    const ExternalUint8Array& data_;
+    ReadStream stream_;
     Array& token_objects_;
     Object& obj_;
     intptr_t cur_token_pos_;
-    intptr_t stream_token_pos_;
     Token::Kind cur_token_kind_;
     intptr_t cur_token_obj_index_;
   };
 
  private:
-  void SetLength(intptr_t value) const;
-
   RawString* PrivateKey() const;
   void SetPrivateKey(const String& value) const;
 
-  uint8_t* EntryAddr(intptr_t token_pos) const {
-    ASSERT((token_pos >=0) && (token_pos < Length()));
-    return &raw_ptr()->data_[token_pos];
-  }
+  static RawTokenStream* New();
+  static void DataFinalizer(void *peer);
 
   HEAP_OBJECT_IMPLEMENTATION(TokenStream, Object);
   friend class Class;
@@ -4834,6 +4829,7 @@
   HEAP_OBJECT_IMPLEMENTATION(ExternalUint8Array, ByteArray);
   friend class ByteArray;
   friend class Class;
+  friend class TokenStream;
 };
 
 
@@ -5534,11 +5530,6 @@
 }
 
 
-intptr_t TokenStream::Length() const {
-  return Smi::Value(raw_ptr()->length_);
-}
-
-
 void Context::SetAt(intptr_t index, const Instance& value) const {
   StorePointer(InstanceAddr(index), value.raw());
 }