Use external byte arrays for token stream, this moves the token stream out of the isolate heap. For…

vm/object.h

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 #ifndef VM_OBJECT_H_
 #define VM_OBJECT_H_
 
 #include "include/dart_api.h"
 #include "platform/assert.h"
 #include "platform/utils.h"
@@ skipping 1804 matching lines @@
   void set_literal(const String& literal) const;
   void set_value(const Object& value) const;
 
   HEAP_OBJECT_IMPLEMENTATION(LiteralToken, Object);
   friend class Class;
 };
 
 
 class TokenStream : public Object {
  public:
-  inline intptr_t Length() const;
-
   RawArray* TokenObjects() const;
   void SetTokenObjects(const Array& value) const;
 
+  RawExternalUint8Array* GetStream() const;
+  void SetStream(const ExternalUint8Array& stream) const;
+
   RawString* GenerateSource() const;
   intptr_t ComputeSourcePosition(intptr_t tok_pos) const;
   intptr_t ComputeTokenPosition(intptr_t src_pos) const;
 
   static const intptr_t kBytesPerElement = 1;
   static const intptr_t kMaxElements = kSmiMax / kBytesPerElement;
 
   static intptr_t InstanceSize() {
-    ASSERT(sizeof(RawTokenStream) == OFFSET_OF(RawTokenStream, data_));
-    return 0;
-  }
-  static intptr_t InstanceSize(intptr_t len) {
-    ASSERT(0 <= len && len <= kMaxElements);
-    return RoundedAllocationSize(
-        sizeof(RawTokenStream) + (len * kBytesPerElement));
+    return RoundedAllocationSize(sizeof(RawTokenStream));
   }
 
   static RawTokenStream* New(intptr_t length);
   static RawTokenStream* New(const Scanner::GrowableTokenStream& tokens,
                              const String& private_key);
 
   // The class Iterator encapsulates iteration over the tokens
   // in a TokenStream object.
   class Iterator : ValueObject {
    public:
@@ skipping 12 matching lines @@
 
     void Advance();
 
     RawObject* CurrentToken() const;
     RawString* CurrentLiteral() const;
     RawString* MakeLiteralToken(const Object& obj) const;
 
    private:
     // Read token from the token stream (could be a simple token or an index
     // into the token objects array for IDENT or literal tokens).
-    intptr_t ReadToken();
-    uint8_t ReadByte();
+    intptr_t ReadToken() {
+      int64_t value = stream_.ReadUnsigned();
+      ASSERT((value >= 0) && (value <= kIntptrMax));
+      return value;
+    }
 
     const TokenStream& tokens_;
+    const ExternalUint8Array& data_;
+    ReadStream stream_;
     Array& token_objects_;
     Object& obj_;
     intptr_t cur_token_pos_;
-    intptr_t stream_token_pos_;
     Token::Kind cur_token_kind_;
     intptr_t cur_token_obj_index_;
   };
 
  private:
-  void SetLength(intptr_t value) const;
-
   RawString* PrivateKey() const;
   void SetPrivateKey(const String& value) const;
 
-  uint8_t* EntryAddr(intptr_t token_pos) const {
-    ASSERT((token_pos >=0) && (token_pos < Length()));
-    return &raw_ptr()->data_[token_pos];
-  }
+  static RawTokenStream* New();
+  static void DataFinalizer(void *peer);
 
   HEAP_OBJECT_IMPLEMENTATION(TokenStream, Object);
   friend class Class;
 };
 
 
 class Script : public Object {
  public:
   RawString* url() const { return raw_ptr()->url_; }
   bool HasSource() const;
@@ skipping 2918 matching lines @@
     return data + byte_offset;
   }
 
   void SetExternalData(ExternalByteArrayData<uint8_t>* data) {
     raw_ptr()->external_data_ = data;
   }
 
   HEAP_OBJECT_IMPLEMENTATION(ExternalUint8Array, ByteArray);
   friend class ByteArray;
   friend class Class;
+  friend class TokenStream;
 };
 
 
 class ExternalInt16Array : public ByteArray {
  public:
   intptr_t ByteLength() const {
     return Length() * kBytesPerElement;
   }
 
   int16_t At(intptr_t index) const {
@@ skipping 680 matching lines @@
   return Smi::Value(reinterpret_cast<RawSmi*>(raw_ptr()->value_));
 }
 
 
 void Field::SetOffset(intptr_t value) const {
   ASSERT(!is_static());  // SetOffset is valid only for instance fields.
   raw_ptr()->value_ = Smi::New(value);
 }
 
 
-intptr_t TokenStream::Length() const {
-  return Smi::Value(raw_ptr()->length_);
-}
-
-
 void Context::SetAt(intptr_t index, const Instance& value) const {
   StorePointer(InstanceAddr(index), value.raw());
 }
 
 
 bool String::Equals(const String& str) const {
   if (raw() == str.raw()) {
     return true;  // Both handles point to the same raw instance.
   }
   if (str.IsNull()) {
@@ skipping 17 matching lines @@
     if (this->CharAt(i) != str.CharAt(begin_index + i)) {
       return false;
     }
   }
   return true;
 }
 
 }  // namespace dart
 
 #endif  // VM_OBJECT_H_