Experiment for updating the tcmalloc chromium branch to r144 (gperftools 2.0).

third_party/tcmalloc/chromium/src/debugallocation.cc

 // Copyright (c) 2000, Google Inc.
 // All rights reserved.
 //
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 // notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
@@ skipping 13 matching lines @@
 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 // ---
 // Author: Urs Holzle <opensource@google.com>
 
 #include "config.h"
+#include <errno.h>
+#ifdef HAVE_FCNTL_H
+#include <fcntl.h>
+#endif
+#ifdef HAVE_INTTYPES_H
+#include <inttypes.h>
+#endif
 // We only need malloc.h for struct mallinfo.
 #ifdef HAVE_STRUCT_MALLINFO
 // Malloc can be in several places on older versions of OS X.
 # if defined(HAVE_MALLOC_H)
 # include <malloc.h>
 # elif defined(HAVE_MALLOC_MALLOC_H)
 # include <malloc/malloc.h>
 # elif defined(HAVE_SYS_MALLOC_H)
 # include <sys/malloc.h>
 # endif
 #endif
+#ifdef HAVE_PTHREAD
 #include <pthread.h>
-#include <stdio.h>
-#ifdef HAVE_INTTYPES_H
-#include <inttypes.h>
 #endif
 #include <stdarg.h>
+#include <stdio.h>
+#include <string.h>
 #ifdef HAVE_MMAP
 #include <sys/mman.h>
 #endif
+#include <sys/stat.h>
 #include <sys/types.h>
-#include <sys/stat.h>
-#ifdef HAVE_FCNTL_H
-#include <fcntl.h>
-#endif
 #ifdef HAVE_UNISTD_H
 #include <unistd.h>
 #endif
-#include <errno.h>
-#include <string.h>
 
-#include <google/malloc_extension.h>
-#include <google/malloc_hook.h>
-#include <google/stacktrace.h>
+#include <gperftools/malloc_extension.h>
+#include <gperftools/malloc_hook.h>
+#include <gperftools/stacktrace.h>
+#include "addressmap-inl.h"
 #include "base/commandlineflags.h"
 #include "base/googleinit.h"
 #include "base/logging.h"
 #include "base/spinlock.h"
-#include "addressmap-inl.h"
 #include "malloc_hook-inl.h"
 #include "symbolize.h"
 
 #define TCMALLOC_USING_DEBUGALLOCATION
 #include "tcmalloc.cc"
 
 // __THROW is defined in glibc systems.  It means, counter-intuitively,
 // "This function will never throw an exception."  It's an optional
 // optimization tool, but we may need to use it to match glibc prototypes.
 #ifndef __THROW    // I guess we're not on a glibc system
@@ skipping 34 matching lines @@
              EnvToInt("TCMALLOC_MAX_FREE_QUEUE_SIZE", 10*1024*1024),
              "If greater than 0, keep freed blocks in a queue instead of "
              "releasing them to the allocator immediately.  Release them when "
              "the total size of all blocks in the queue would otherwise exceed "
              "this limit.");
 
 DEFINE_bool(symbolize_stacktrace,
             EnvToBool("TCMALLOC_SYMBOLIZE_STACKTRACE", true),
             "Symbolize the stack trace when provided (on some error exits)");
 
+// If we are LD_PRELOAD-ed against a non-pthreads app, then
+// pthread_once won't be defined.  We declare it here, for that
+// case (with weak linkage) which will cause the non-definition to
+// resolve to NULL.  We can then check for NULL or not in Instance.
+extern "C" int pthread_once(pthread_once_t *, void (*)(void))
+    ATTRIBUTE_WEAK;
+
 // ========================================================================= //
 
 // A safe version of printf() that does not do any allocation and
 // uses very little stack space.
 static void TracePrintf(int fd, const char *fmt, ...)
   __attribute__ ((__format__ (__printf__, 2, 3)));
 
 // The do_* functions are defined in tcmalloc/tcmalloc.cc,
 // which is included before this file
 // when TCMALLOC_FOR_DEBUGALLOCATION is defined
-#define BASE_MALLOC_NEW(size)    cpp_alloc(size, false)
+// TODO(csilvers): get rid of these now that we are tied to tcmalloc.
+#define BASE_MALLOC_NEW    do_malloc
 #define BASE_MALLOC        do_malloc
 #define BASE_FREE          do_free
 #define BASE_MALLOC_STATS  do_malloc_stats
 #define BASE_MALLOPT       do_mallopt
 #define BASE_MALLINFO      do_mallinfo
-#define BASE_MALLOC_SIZE(ptr) GetSizeWithCallback(ptr, &InvalidGetAllocatedSize)
 
 // ========================================================================= //
 
 class MallocBlock;
 
 // A circular buffer to hold freed blocks of memory.  MallocBlock::Deallocate
 // (below) pushes blocks into this queue instead of returning them to the
 // underlying allocator immediately.  See MallocBlock::Deallocate for more
 // information.
 //
 // We can't use an STL class for this because we need to be careful not to
 // perform any heap de-allocations in any of the code in this class, since the
 // code in MallocBlock::Deallocate is not re-entrant.
 template <typename QueueEntry>
 class FreeQueue {
  public:
   FreeQueue() : q_front_(0), q_back_(0) {}
 
   bool Full() {
     return (q_front_ + 1) % kFreeQueueSize == q_back_;
   }
 
   void Push(const QueueEntry& block) {
     q_[q_front_] = block;
     q_front_ = (q_front_ + 1) % kFreeQueueSize;
   }
 
   QueueEntry Pop() {
+    RAW_CHECK(q_back_ != q_front_, "Queue is empty");
     const QueueEntry& ret = q_[q_back_];
     q_back_ = (q_back_ + 1) % kFreeQueueSize;
     return ret;
   }
 
   size_t size() const {
     return (q_front_ - q_back_ + kFreeQueueSize) % kFreeQueueSize;
   }
 
  private:
   // Maximum number of blocks kept in the free queue before being freed.
   static const int kFreeQueueSize = 1024;
 
   QueueEntry q_[kFreeQueueSize];
   int q_front_;
   int q_back_;
 };
 
 struct MallocBlockQueueEntry {
   MallocBlockQueueEntry() : block(NULL), size(0),
                             num_deleter_pcs(0), deleter_threadid(0) {}
   MallocBlockQueueEntry(MallocBlock* b, size_t s) : block(b), size(s) {
-    if (FLAGS_max_free_queue_size != 0) {
+    if (FLAGS_max_free_queue_size != 0 && b != NULL) {
       // Adjust the number of frames to skip (4) if you change the
       // location of this call.
       num_deleter_pcs =
           GetStackTrace(deleter_pcs,
                         sizeof(deleter_pcs) / sizeof(deleter_pcs[0]),
                         4);
       deleter_threadid = pthread_self();
     } else {
       num_deleter_pcs = 0;
       // Zero is an illegal pthread id by my reading of the pthread
@@ skipping 54 matching lines @@
   // start the program (see man malloc).
 
   // We use either BASE_MALLOC or mmap to make the actual allocation. In
   // order to remember which one of the two was used for any block, we store an
   // appropriate magic word next to the block.
   static const int kMagicMalloc = 0xDEADBEEF;
   static const int kMagicMMap = 0xABCDEFAB;
 
   // This array will be filled with 0xCD, for use with memcmp.
   static unsigned char kMagicDeletedBuffer[1024];
-  static bool deleted_buffer_initialized_;
+  static pthread_once_t deleted_buffer_initialized_;
+  static bool deleted_buffer_initialized_no_pthreads_;
 
  private:  // data layout
 
                     // The four fields size1_,offset_,magic1_,alloc_type_
                     // should together occupy a multiple of 16 bytes. (At the
                     // moment, sizeof(size_t) == 4 or 8 depending on piii vs
                     // k8, and 4 of those sum to 16 or 32 bytes).
                     // This, combined with BASE_MALLOC's alignment guarantees,
                     // ensures that SSE types can be stored into the returned
                     // block, at &size2_.
@@ skipping 274 matching lines @@
     }
   }
 
   static size_t FreeQueueSize() {
     SpinLockHolder l(&free_queue_lock_);
     return free_queue_size_;
   }
 
   static void ProcessFreeQueue(MallocBlock* b, size_t size,
                                int max_free_queue_size) {
-    SpinLockHolder l(&free_queue_lock_);
+    // MallocBlockQueueEntry are about 144 in size, so we can only
+    // use a small array of them on the stack.
+    MallocBlockQueueEntry entries[4];
+    int num_entries = 0;
+    MallocBlockQueueEntry new_entry(b, size);
+    free_queue_lock_.Lock();
     if (free_queue_ == NULL)
       free_queue_ = new FreeQueue<MallocBlockQueueEntry>;
     RAW_CHECK(!free_queue_->Full(), "Free queue mustn't be full!");
 
     if (b != NULL) {
       free_queue_size_ += size + sizeof(MallocBlockQueueEntry);
-      MallocBlockQueueEntry new_entry(b, size);
       free_queue_->Push(new_entry);
     }
 
     // Free blocks until the total size of unfreed blocks no longer exceeds
     // max_free_queue_size, and the free queue has at least one free
     // space in it.
     while (free_queue_size_ > max_free_queue_size || free_queue_->Full()) {
-      MallocBlockQueueEntry cur = free_queue_->Pop();
-      CheckForDanglingWrites(cur);
-      free_queue_size_ -= cur.size + sizeof(MallocBlockQueueEntry);
-      BASE_FREE(cur.block);
+      RAW_CHECK(num_entries < arraysize(entries), "entries array overflow");
+      entries[num_entries] = free_queue_->Pop();
+      free_queue_size_ -=
+          entries[num_entries].size + sizeof(MallocBlockQueueEntry);
+      num_entries++;
+      if (num_entries == arraysize(entries)) {
+        // The queue will not be full at this point, so it is ok to
+        // release the lock.  The queue may still contain more than
+        // max_free_queue_size, but this is not a strict invariant.
+        free_queue_lock_.Unlock();
+        for (int i = 0; i < num_entries; i++) {
+          CheckForDanglingWrites(entries[i]);
+          BASE_FREE(entries[i].block);
+        }
+        num_entries = 0;
+        free_queue_lock_.Lock();
+      }
     }
     RAW_CHECK(free_queue_size_ >= 0, "Free queue size went negative!");
+    free_queue_lock_.Unlock();
+    for (int i = 0; i < num_entries; i++) {
+      CheckForDanglingWrites(entries[i]);
+      BASE_FREE(entries[i].block);
+    }
+  }
+
+  static void InitDeletedBuffer() {
+    memset(kMagicDeletedBuffer, kMagicDeletedByte, sizeof(kMagicDeletedBuffer));
+    deleted_buffer_initialized_no_pthreads_ = true;
   }
 
   static void CheckForDanglingWrites(const MallocBlockQueueEntry& queue_entry) {
     // Initialize the buffer if necessary.
-    if (!deleted_buffer_initialized_) {
-      // This is threadsafe.  We hold free_queue_lock_.
-      memset(kMagicDeletedBuffer, 0xcd, sizeof(kMagicDeletedBuffer));
-      deleted_buffer_initialized_ = true;
+    if (pthread_once)
+      pthread_once(&deleted_buffer_initialized_, &InitDeletedBuffer);
+    if (!deleted_buffer_initialized_no_pthreads_) {
+      // This will be the case on systems that don't link in pthreads,
+      // including on FreeBSD where pthread_once has a non-zero address
+      // (but doesn't do anything) even when pthreads isn't linked in.
+      InitDeletedBuffer();
     }
 
     const unsigned char* p =
         reinterpret_cast<unsigned char*>(queue_entry.block);
 
     static const size_t size_of_buffer = sizeof(kMagicDeletedBuffer);
     const size_t size = queue_entry.size;
     const size_t buffers = size / size_of_buffer;
     const size_t remainder = size % size_of_buffer;
     size_t buffer_idx;
@@ skipping 15 matching lines @@
     RAW_LOG(ERROR,
             "Found a corrupted memory buffer in MallocBlock (may be offset "
             "from user ptr): buffer index: %zd, buffer ptr: %p, size of "
             "buffer: %zd", buffer_idx, buffer, size_of_buffer);
 
     // The magic deleted buffer should only be 1024 bytes, but in case
     // this changes, let's put an upper limit on the number of debug
     // lines we'll output:
     if (size_of_buffer <= 1024) {
       for (int i = 0; i < size_of_buffer; ++i) {
-        if (buffer[i] != 0xcd) {
-          RAW_LOG(ERROR, "Buffer byte %d is 0x%02x (should be 0xcd).",
-                  i, buffer[i]);
+        if (buffer[i] != kMagicDeletedByte) {
+          RAW_LOG(ERROR, "Buffer byte %d is 0x%02x (should be 0x%02x).",
+                  i, buffer[i], kMagicDeletedByte);
         }
       }
     } else {
       RAW_LOG(ERROR, "Buffer too large to print corruption.");
     }
 
     const MallocBlock* b = queue_entry.block;
     const size_t size = queue_entry.size;
     if (queue_entry.num_deleter_pcs > 0) {
       TracePrintf(STDERR_FILENO, "Deleted by thread %p\n",
@@ skipping 23 matching lines @@
       RAW_LOG(ERROR,
               "Skipping the printing of the deleter's stack!  Its stack was "
               "not found; either the corruption occurred too early in "
               "execution to obtain a stack trace or --max_free_queue_size was "
               "set to 0.");
     }
 
     RAW_LOG(FATAL,
             "Memory was written to after being freed.  MallocBlock: %p, user "
             "ptr: %p, size: %zd.  If you can't find the source of the error, "
-            "try using valgrind or purify, or study the output of the "
-            "deleter's stack printed above.", b, b->data_addr(), size);
+            "try using ASan (http://code.google.com/p/address-sanitizer/), "
+            "Valgrind, or Purify, or study the "
+            "output of the deleter's stack printed above.",
+            b, b->data_addr(), size);
   }
 
   static MallocBlock* FromRawPointer(void* p) {
     const size_t data_offset = MallocBlock::data_offset();
     // Find the header just before client's memory.
     MallocBlock *mb = reinterpret_cast<MallocBlock *>(
                 reinterpret_cast<char *>(p) - data_offset);
     // If mb->alloc_type_ is kMagicDeletedSizeT, we're not an ok pointer.
     if (mb->alloc_type_ == kMagicDeletedSizeT) {
       RAW_LOG(FATAL, "memory allocation bug: object at %p has been already"
                      " deallocated; or else a word before the object has been"
                      " corrupted (memory stomping bug)", p);
     }
     // If mb->offset_ is zero (common case), mb is the real header.  If
     // mb->offset_ is non-zero, this block was allocated by memalign, and
     // mb->offset_ is the distance backwards to the real header from mb,
     // which is a fake header.  The following subtraction works for both zero
     // and non-zero values.
     return reinterpret_cast<MallocBlock *>(
                 reinterpret_cast<char *>(mb) - mb->offset_);
   }
   static const MallocBlock* FromRawPointer(const void* p) {
     // const-safe version: we just cast about
     return FromRawPointer(const_cast<void*>(p));
   }
 
-  void Check(int type) {
+  void Check(int type) const {
     alloc_map_lock_.Lock();
     CheckLocked(type);
     alloc_map_lock_.Unlock();
   }
 
   static bool CheckEverything() {
     alloc_map_lock_.Lock();
     if (alloc_map_ != NULL)  alloc_map_->Iterate(CheckCallback, 0);
     alloc_map_lock_.Unlock();
     return true;  // if we get here, we're okay
@@ skipping 57 matching lines @@
 const int MallocBlock::kMagicMMap;
 
 MallocBlock::AllocMap* MallocBlock::alloc_map_ = NULL;
 SpinLock MallocBlock::alloc_map_lock_(SpinLock::LINKER_INITIALIZED);
 
 FreeQueue<MallocBlockQueueEntry>* MallocBlock::free_queue_ = NULL;
 size_t MallocBlock::free_queue_size_ = 0;
 SpinLock MallocBlock::free_queue_lock_(SpinLock::LINKER_INITIALIZED);
 
 unsigned char MallocBlock::kMagicDeletedBuffer[1024];
-bool MallocBlock::deleted_buffer_initialized_ = false;
+pthread_once_t MallocBlock::deleted_buffer_initialized_ = PTHREAD_ONCE_INIT;
+bool MallocBlock::deleted_buffer_initialized_no_pthreads_ = false;
 
 const char* const MallocBlock::kAllocName[] = {
   "malloc",
   "new",
   "new []",
   NULL,
 };
 
 const char* const MallocBlock::kDeallocName[] = {
   "free",
@@ skipping 173 matching lines @@
     if (result && (strcmp(name, "generic.current_allocated_bytes") == 0)) {
       // Subtract bytes kept in the free queue
       size_t qsize = MallocBlock::FreeQueueSize();
       if (*value >= qsize) {
         *value -= qsize;
       }
     }
     return result;
   }
 
-  virtual bool VerifyNewMemory(void* p) {
+  virtual bool VerifyNewMemory(const void* p) {
     if (p)  MallocBlock::FromRawPointer(p)->Check(MallocBlock::kNewType);
     return true;
   }
 
-  virtual bool VerifyArrayNewMemory(void* p) {
+  virtual bool VerifyArrayNewMemory(const void* p) {
     if (p)  MallocBlock::FromRawPointer(p)->Check(MallocBlock::kArrayNewType);
     return true;
   }
 
-  virtual bool VerifyMallocMemory(void* p) {
+  virtual bool VerifyMallocMemory(const void* p) {
     if (p)  MallocBlock::FromRawPointer(p)->Check(MallocBlock::kMallocType);
     return true;
   }
 
   virtual bool VerifyAllMemory() {
     return MallocBlock::CheckEverything();
   }
 
   virtual bool MallocMemoryStats(int* blocks, size_t* total,
                                  int histogram[kMallocHistogramSize]) {
     return MallocBlock::MemoryStats(blocks, total, histogram);
   }
 
-  virtual size_t GetAllocatedSize(void* p) {
+  virtual size_t GetEstimatedAllocatedSize(size_t size) {
+    return size;
+  }
+
+  virtual size_t GetAllocatedSize(const void* p) {
     if (p) {
+      RAW_CHECK(GetOwnership(p) != MallocExtension::kNotOwned,
+                "ptr not allocated by tcmalloc");
       return MallocBlock::FromRawPointer(p)->data_size();
     }
     return 0;
   }
-  virtual size_t GetEstimatedAllocatedSize(size_t size) {
-    return size;
+
+  virtual MallocExtension::Ownership GetOwnership(const void* p) {
+    if (p) {
+      const MallocBlock* mb = MallocBlock::FromRawPointer(p);
+      return TCMallocImplementation::GetOwnership(mb);
+    }
+    return MallocExtension::kNotOwned;   // nobody owns NULL
   }
 
   virtual void GetFreeListSizes(vector<MallocExtension::FreeListInfo>* v) {
     static const char* kDebugFreeQueue = "debug.free_queue";
 
     TCMallocImplementation::GetFreeListSizes(v);
 
     MallocExtension::FreeListInfo i;
     i.type = kDebugFreeQueue;
     i.min_object_size = 0;
     i.max_object_size = numeric_limits<size_t>::max();
     i.total_bytes_free = MallocBlock::FreeQueueSize();
     v->push_back(i);
   }
 
  };
 
 static DebugMallocImplementation debug_malloc_implementation;
 
 REGISTER_MODULE_INITIALIZER(debugallocation, {
   // Either we or valgrind will control memory management.  We
-  // register our extension if we're the winner.
-  if (RunningOnValgrind()) {
-    // Let Valgrind uses its own malloc (so don't register our extension).
-  } else {
+  // register our extension if we're the winner. Otherwise let
+  // Valgrind use its own malloc (so don't register our extension).
+  if (!RunningOnValgrind()) {
     MallocExtension::Register(&debug_malloc_implementation);
+  }
+});
+
+REGISTER_MODULE_DESTRUCTOR(debugallocation, {
+  if (!RunningOnValgrind()) {
     // When the program exits, check all blocks still in the free
     // queue for corruption.
-    atexit(DanglingWriteChecker);
+    DanglingWriteChecker();
   }
 });
 
 // ========================================================================= //
 
 // This is mostly the same a cpp_alloc in tcmalloc.cc.
-// TODO(csilvers): write a wrapper for new-handler so we don't have to
-// copy this code so much.
+// TODO(csilvers): change Allocate() above to call cpp_alloc, so we
+// don't have to reproduce the logic here.  To make tc_new_mode work
+// properly, I think we'll need to separate out the logic of throwing
+// from the logic of calling the new-handler.
 inline void* debug_cpp_alloc(size_t size, int new_type, bool nothrow) {
   for (;;) {
     void* p = DebugAllocate(size, new_type);
 #ifdef PREANSINEW
     return p;
 #else
     if (p == NULL) {  // allocation failed
       // Get the current new handler.  NB: this function is not
       // thread-safe.  We make a feeble stab at making it so here, but
       // this lock only protects against tcmalloc interfering with
@@ skipping 295 matching lines @@
   return BASE_MALLOPT(cmd, value);
 }
 
 #ifdef HAVE_STRUCT_MALLINFO
 extern "C" PERFTOOLS_DLL_DECL struct mallinfo tc_mallinfo(void) __THROW {
   return BASE_MALLINFO();
 }
 #endif
 
 extern "C" PERFTOOLS_DLL_DECL size_t tc_malloc_size(void* ptr) __THROW {
-  if (!ptr) {
-    return 0;
-  }
-  MallocBlock* mb = MallocBlock::FromRawPointer(ptr);
-  // This is just to make sure we actually own mb (and ptr).  We don't
-  // use the actual value, just the 'exception' it raises on error.
-  (void)BASE_MALLOC_SIZE(mb);
-  return mb->data_size();
+  return MallocExtension::instance()->GetAllocatedSize(ptr);
 }
-
-// Override __libc_memalign in libc on linux boxes.
-// They have a bug in libc that causes them (very rarely) to allocate
-// with __libc_memalign() yet deallocate with free().
-// This function is an exception to the rule of calling MallocHook method
-// from the stack frame of the allocation function;
-// heap-checker handles this special case explicitly.
-static void *MemalignOverride(size_t align, size_t size, const void *caller)
-    __THROW ATTRIBUTE_SECTION(google_malloc);
-
-static void *MemalignOverride(size_t align, size_t size, const void *caller)
-    __THROW {
-  void *p = do_debug_memalign_or_debug_cpp_memalign(align, size);
-  MallocHook::InvokeNewHook(p, size);
-  return p;
-}
-void *(*__memalign_hook)(size_t, size_t, const void *) = MemalignOverride;