V8 crashes when profile generator is trying to allocate a raw_entries_ buffer more than 2Gb.

src/profile-generator.cc

 // Copyright 2012 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 1078 matching lines @@
     case kArray: return "/array/";
     case kRegExp: return "/regexp/";
     case kHeapNumber: return "/number/";
     case kNative: return "/native/";
     case kSynthetic: return "/synthetic/";
     default: return "???";
   }
 }
 
 
-int HeapEntry::EntriesSize(int entries_count,
+uint64_t HeapEntry::EntriesSize(int entries_count,
                            int children_count,
                            int retainers_count) {
-  return sizeof(HeapEntry) * entries_count         // NOLINT
-      + sizeof(HeapGraphEdge) * children_count     // NOLINT
-      + sizeof(HeapGraphEdge*) * retainers_count;  // NOLINT
+  return (uint64_t)sizeof(HeapEntry) * entries_count         // NOLINT

[mnaganov (inactive), 2012/03/08 15:25:10]

Please, use static_cast.

[alexeif, 2012/03/10 10:32:02]

There's no need to use casts here. sizeof has the type of size_t which has the
width of the pointer type. Just change the return type of EntriesSize to size_t
and that should be enough.

+      + (uint64_t)sizeof(HeapGraphEdge) * children_count     // NOLINT
+      + (uint64_t)sizeof(HeapGraphEdge*) * retainers_count;  // NOLINT
 }
 
 
 // It is very important to keep objects that form a heap snapshot
 // as small as possible.
 namespace {  // Avoid littering the global namespace.
 
 template <size_t ptr_size> struct SnapshotSizeConstants;
 
 template <> struct SnapshotSizeConstants<4> {
   static const int kExpectedHeapGraphEdgeSize = 12;
   static const int kExpectedHeapEntrySize = 36;
-  static const int kMaxSerializableSnapshotRawSize = 256 * MB;
+  static const size_t kMaxSerializableSnapshotRawSize = 256 * MB;
 };
 
 template <> struct SnapshotSizeConstants<8> {
   static const int kExpectedHeapGraphEdgeSize = 24;
   static const int kExpectedHeapEntrySize = 48;
-  static const int kMaxSerializableSnapshotRawSize = 768 * MB;
+  static const size_t kMaxSerializableSnapshotRawSize = 6000ul * MB;
 };
 
 }  // namespace
 
+class RawEntriesStorage {
+ public:
+  explicit RawEntriesStorage(size_t size) : size_(size) { }
+  ~RawEntriesStorage() {
+    for (int i = 0; i < raw_data_.length(); ++i)
+      DeleteArray(raw_data_[i]);
+  }
+  char* FirstAddress(size_t minimum_size) {

[mnaganov (inactive), 2012/03/08 15:25:10]

I think, you should use Address type.

+    ASSERT(raw_data_.length() == 0);
+    return AllocateNextChunk(minimum_size);
+  }
+  char* NextAddress(char* next_entry_candidate, size_t minimum_size) {
+    ASSERT(raw_data_.length());

[mnaganov (inactive), 2012/03/08 15:25:10]

!= 0

+    if (next_entry_candidate - last_chunk_ + minimum_size > last_chunk_size_) {

[mnaganov (inactive), 2012/03/08 15:25:10]

Note: Please verify that expressions like these compile both on ia32, x64 and on
Windows.

+      return AllocateNextChunk(minimum_size);
+    }
+    return next_entry_candidate;
+  }
+
+ private:
+  char* AllocateNextChunk(size_t minimum_size) {
+    last_chunk_size_ =
+        default_chunk_size_ > minimum_size ? default_chunk_size_ : minimum_size;
+    last_chunk_ = NewArray<char>(last_chunk_size_);

[alexeif, 2012/03/10 10:32:02]

Can't we just change the NewArray argument to size_t instead of workarounding it
with such a kludge?

+    raw_data_.Add(last_chunk_);
+    return last_chunk_;
+  }
+  size_t size_;
+  size_t last_chunk_size_;
+  char* last_chunk_;
+  List<char*> raw_data_;
+  static const size_t default_chunk_size_ = 256 * MB;
+};
+
 HeapSnapshot::HeapSnapshot(HeapSnapshotsCollection* collection,
                            HeapSnapshot::Type type,
                            const char* title,
                            unsigned uid)
     : collection_(collection),
       type_(type),
       title_(title),
       uid_(uid),
       root_entry_(NULL),
       gc_roots_entry_(NULL),
       natives_root_entry_(NULL),
       raw_entries_(NULL),
       entries_sorted_(false) {
   STATIC_ASSERT(
       sizeof(HeapGraphEdge) ==
       SnapshotSizeConstants<kPointerSize>::kExpectedHeapGraphEdgeSize);
   STATIC_ASSERT(
       sizeof(HeapEntry) ==
       SnapshotSizeConstants<kPointerSize>::kExpectedHeapEntrySize);
   for (int i = 0; i < VisitorSynchronization::kNumberOfSyncTags; ++i) {
     gc_subroot_entries_[i] = NULL;
   }
 }
 
 
 HeapSnapshot::~HeapSnapshot() {
-  DeleteArray(raw_entries_);
+  delete raw_entries_;
 }
 
 
 void HeapSnapshot::Delete() {
   collection_->RemoveSnapshot(this);
   delete this;
 }
 
 
 void HeapSnapshot::AllocateEntries(int entries_count,
                                    int children_count,
                                    int retainers_count) {
   ASSERT(raw_entries_ == NULL);
   raw_entries_size_ =
       HeapEntry::EntriesSize(entries_count, children_count, retainers_count);
-  raw_entries_ = NewArray<char>(raw_entries_size_);
+  raw_entries_ = new RawEntriesStorage(raw_entries_size_);
 }
 
 
 static void HeapEntryClearPaint(HeapEntry** entry_ptr) {
   (*entry_ptr)->clear_paint();
 }
 
 
 void HeapSnapshot::ClearPaint() {
   entries_.Iterate(HeapEntryClearPaint);
@@ skipping 37 matching lines @@
       retainers_count));
 }
 
 
 HeapEntry* HeapSnapshot::AddEntry(HeapEntry::Type type,
                                   const char* name,
                                   uint64_t id,
                                   int size,
                                   int children_count,
                                   int retainers_count) {
-  HeapEntry* entry = GetNextEntryToInit();
+  HeapEntry* entry = GetNextEntryToInit(
+      HeapEntry::EntriesSize(1, children_count, retainers_count));
   entry->Init(this, type, name, id, size, children_count, retainers_count);
   return entry;
 }
 
 
 void HeapSnapshot::SetDominatorsToSelf() {
   for (int i = 0; i < entries_.length(); ++i) {
     HeapEntry* entry = entries_[i];
     if (entry->dominator() == NULL) entry->set_dominator(entry);
   }
 }
 
 
-HeapEntry* HeapSnapshot::GetNextEntryToInit() {
+HeapEntry* HeapSnapshot::GetNextEntryToInit(size_t size) {
   if (entries_.length() > 0) {
     HeapEntry* last_entry = entries_.last();
+    HeapEntry* next_entry = reinterpret_cast<HeapEntry*>(
+        raw_entries_->NextAddress(reinterpret_cast<char*>(last_entry)
+            + last_entry->EntrySize(), size));
+    entries_.Add(next_entry);
+  } else {
     entries_.Add(reinterpret_cast<HeapEntry*>(
-        reinterpret_cast<char*>(last_entry) + last_entry->EntrySize()));
-  } else {
-    entries_.Add(reinterpret_cast<HeapEntry*>(raw_entries_));
+        raw_entries_->FirstAddress(size)));
   }
-  ASSERT(reinterpret_cast<char*>(entries_.last()) <
-         (raw_entries_ + raw_entries_size_));
   return entries_.last();
 }
 
 
 HeapEntry* HeapSnapshot::GetEntryById(uint64_t id) {
   List<HeapEntry*>* entries_by_id = GetSortedEntriesList();
 
   // Perform a binary search by id.
   int low = 0;
   int high = entries_by_id->length() - 1;
@@ skipping 2167 matching lines @@
 
 HeapSnapshot* HeapSnapshotJSONSerializer::CreateFakeSnapshot() {
   HeapSnapshot* result = new HeapSnapshot(snapshot_->collection(),
                                           HeapSnapshot::kFull,
                                           snapshot_->title(),
                                           snapshot_->uid());
   result->AllocateEntries(2, 1, 0);
   HeapEntry* root = result->AddRootEntry(1);
   const char* text = snapshot_->collection()->names()->GetFormatted(
       "The snapshot is too big. "
       "Maximum snapshot size is %d MB. "

[mnaganov (inactive), 2012/03/08 15:25:10]

Is %d the correct type?

-      "Actual snapshot size is %d MB.",
+      "Actual snapshot size is %u MB.",

[alexeif, 2012/03/10 10:32:02]

if raw_entries_size() is uint64_t then it should be %llu. If you change it to
size_t then use %lu.

       SnapshotSizeConstants<kPointerSize>::kMaxSerializableSnapshotRawSize / MB,
       (snapshot_->raw_entries_size() + MB - 1) / MB);
   HeapEntry* message = result->AddEntry(
       HeapEntry::kString, text, 0, 4, 0, 0);
   root->SetUnidirElementReference(0, 1, message);
   result->SetDominatorsToSelf();
   return result;
 }
 
 
@@ skipping 283 matching lines @@
 
 
 void HeapSnapshotJSONSerializer::SortHashMap(
     HashMap* map, List<HashMap::Entry*>* sorted_entries) {
   for (HashMap::Entry* p = map->Start(); p != NULL; p = map->Next(p))
     sorted_entries->Add(p);
   sorted_entries->Sort(SortUsingEntryValue);
 }
 
 } }  // namespace v8::internal