OLD | NEW |
1 // Copyright (c) 2000, Google Inc. | 1 // Copyright (c) 2000, Google Inc. |
2 // All rights reserved. | 2 // All rights reserved. |
3 // | 3 // |
4 // Redistribution and use in source and binary forms, with or without | 4 // Redistribution and use in source and binary forms, with or without |
5 // modification, are permitted provided that the following conditions are | 5 // modification, are permitted provided that the following conditions are |
6 // met: | 6 // met: |
7 // | 7 // |
8 // * Redistributions of source code must retain the above copyright | 8 // * Redistributions of source code must retain the above copyright |
9 // notice, this list of conditions and the following disclaimer. | 9 // notice, this list of conditions and the following disclaimer. |
10 // * Redistributions in binary form must reproduce the above | 10 // * Redistributions in binary form must reproduce the above |
(...skipping 13 matching lines...) Expand all Loading... |
24 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | 24 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
25 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | 25 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
26 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | 26 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
27 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | 27 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
28 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | 28 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
29 | 29 |
30 // --- | 30 // --- |
31 // Author: Urs Holzle <opensource@google.com> | 31 // Author: Urs Holzle <opensource@google.com> |
32 | 32 |
33 #include "config.h" | 33 #include "config.h" |
34 #include <errno.h> | |
35 #ifdef HAVE_FCNTL_H | |
36 #include <fcntl.h> | |
37 #endif | |
38 #ifdef HAVE_INTTYPES_H | |
39 #include <inttypes.h> | |
40 #endif | |
41 // We only need malloc.h for struct mallinfo. | 34 // We only need malloc.h for struct mallinfo. |
42 #ifdef HAVE_STRUCT_MALLINFO | 35 #ifdef HAVE_STRUCT_MALLINFO |
43 // Malloc can be in several places on older versions of OS X. | 36 // Malloc can be in several places on older versions of OS X. |
44 # if defined(HAVE_MALLOC_H) | 37 # if defined(HAVE_MALLOC_H) |
45 # include <malloc.h> | 38 # include <malloc.h> |
46 # elif defined(HAVE_MALLOC_MALLOC_H) | 39 # elif defined(HAVE_MALLOC_MALLOC_H) |
47 # include <malloc/malloc.h> | 40 # include <malloc/malloc.h> |
48 # elif defined(HAVE_SYS_MALLOC_H) | 41 # elif defined(HAVE_SYS_MALLOC_H) |
49 # include <sys/malloc.h> | 42 # include <sys/malloc.h> |
50 # endif | 43 # endif |
51 #endif | 44 #endif |
52 #ifdef HAVE_PTHREAD | |
53 #include <pthread.h> | 45 #include <pthread.h> |
| 46 #include <stdio.h> |
| 47 #ifdef HAVE_INTTYPES_H |
| 48 #include <inttypes.h> |
54 #endif | 49 #endif |
55 #include <stdarg.h> | 50 #include <stdarg.h> |
56 #include <stdio.h> | |
57 #include <string.h> | |
58 #ifdef HAVE_MMAP | 51 #ifdef HAVE_MMAP |
59 #include <sys/mman.h> | 52 #include <sys/mman.h> |
60 #endif | 53 #endif |
| 54 #include <sys/types.h> |
61 #include <sys/stat.h> | 55 #include <sys/stat.h> |
62 #include <sys/types.h> | 56 #ifdef HAVE_FCNTL_H |
| 57 #include <fcntl.h> |
| 58 #endif |
63 #ifdef HAVE_UNISTD_H | 59 #ifdef HAVE_UNISTD_H |
64 #include <unistd.h> | 60 #include <unistd.h> |
65 #endif | 61 #endif |
| 62 #include <errno.h> |
| 63 #include <string.h> |
66 | 64 |
67 #include <gperftools/malloc_extension.h> | 65 #include <google/malloc_extension.h> |
68 #include <gperftools/malloc_hook.h> | 66 #include <google/malloc_hook.h> |
69 #include <gperftools/stacktrace.h> | 67 #include <google/stacktrace.h> |
70 #include "addressmap-inl.h" | |
71 #include "base/commandlineflags.h" | 68 #include "base/commandlineflags.h" |
72 #include "base/googleinit.h" | 69 #include "base/googleinit.h" |
73 #include "base/logging.h" | 70 #include "base/logging.h" |
74 #include "base/spinlock.h" | 71 #include "base/spinlock.h" |
| 72 #include "addressmap-inl.h" |
75 #include "malloc_hook-inl.h" | 73 #include "malloc_hook-inl.h" |
76 #include "symbolize.h" | 74 #include "symbolize.h" |
77 | 75 |
78 #define TCMALLOC_USING_DEBUGALLOCATION | 76 #define TCMALLOC_USING_DEBUGALLOCATION |
79 #include "tcmalloc.cc" | 77 #include "tcmalloc.cc" |
80 | 78 |
81 // __THROW is defined in glibc systems. It means, counter-intuitively, | 79 // __THROW is defined in glibc systems. It means, counter-intuitively, |
82 // "This function will never throw an exception." It's an optional | 80 // "This function will never throw an exception." It's an optional |
83 // optimization tool, but we may need to use it to match glibc prototypes. | 81 // optimization tool, but we may need to use it to match glibc prototypes. |
84 #ifndef __THROW // I guess we're not on a glibc system | 82 #ifndef __THROW // I guess we're not on a glibc system |
(...skipping 34 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
119 EnvToInt("TCMALLOC_MAX_FREE_QUEUE_SIZE", 10*1024*1024), | 117 EnvToInt("TCMALLOC_MAX_FREE_QUEUE_SIZE", 10*1024*1024), |
120 "If greater than 0, keep freed blocks in a queue instead of " | 118 "If greater than 0, keep freed blocks in a queue instead of " |
121 "releasing them to the allocator immediately. Release them when " | 119 "releasing them to the allocator immediately. Release them when " |
122 "the total size of all blocks in the queue would otherwise exceed " | 120 "the total size of all blocks in the queue would otherwise exceed " |
123 "this limit."); | 121 "this limit."); |
124 | 122 |
125 DEFINE_bool(symbolize_stacktrace, | 123 DEFINE_bool(symbolize_stacktrace, |
126 EnvToBool("TCMALLOC_SYMBOLIZE_STACKTRACE", true), | 124 EnvToBool("TCMALLOC_SYMBOLIZE_STACKTRACE", true), |
127 "Symbolize the stack trace when provided (on some error exits)"); | 125 "Symbolize the stack trace when provided (on some error exits)"); |
128 | 126 |
129 // If we are LD_PRELOAD-ed against a non-pthreads app, then | |
130 // pthread_once won't be defined. We declare it here, for that | |
131 // case (with weak linkage) which will cause the non-definition to | |
132 // resolve to NULL. We can then check for NULL or not in Instance. | |
133 extern "C" int pthread_once(pthread_once_t *, void (*)(void)) | |
134 ATTRIBUTE_WEAK; | |
135 | |
136 // ========================================================================= // | 127 // ========================================================================= // |
137 | 128 |
138 // A safe version of printf() that does not do any allocation and | 129 // A safe version of printf() that does not do any allocation and |
139 // uses very little stack space. | 130 // uses very little stack space. |
140 static void TracePrintf(int fd, const char *fmt, ...) | 131 static void TracePrintf(int fd, const char *fmt, ...) |
141 __attribute__ ((__format__ (__printf__, 2, 3))); | 132 __attribute__ ((__format__ (__printf__, 2, 3))); |
142 | 133 |
143 // The do_* functions are defined in tcmalloc/tcmalloc.cc, | 134 // The do_* functions are defined in tcmalloc/tcmalloc.cc, |
144 // which is included before this file | 135 // which is included before this file |
145 // when TCMALLOC_FOR_DEBUGALLOCATION is defined | 136 // when TCMALLOC_FOR_DEBUGALLOCATION is defined |
146 // TODO(csilvers): get rid of these now that we are tied to tcmalloc. | 137 #define BASE_MALLOC_NEW(size) cpp_alloc(size, false) |
147 #define BASE_MALLOC_NEW do_malloc | |
148 #define BASE_MALLOC do_malloc | 138 #define BASE_MALLOC do_malloc |
149 #define BASE_FREE do_free | 139 #define BASE_FREE do_free |
150 #define BASE_MALLOC_STATS do_malloc_stats | 140 #define BASE_MALLOC_STATS do_malloc_stats |
151 #define BASE_MALLOPT do_mallopt | 141 #define BASE_MALLOPT do_mallopt |
152 #define BASE_MALLINFO do_mallinfo | 142 #define BASE_MALLINFO do_mallinfo |
| 143 #define BASE_MALLOC_SIZE(ptr) GetSizeWithCallback(ptr, &InvalidGetAllocatedSize) |
153 | 144 |
154 // ========================================================================= // | 145 // ========================================================================= // |
155 | 146 |
156 class MallocBlock; | 147 class MallocBlock; |
157 | 148 |
158 // A circular buffer to hold freed blocks of memory. MallocBlock::Deallocate | 149 // A circular buffer to hold freed blocks of memory. MallocBlock::Deallocate |
159 // (below) pushes blocks into this queue instead of returning them to the | 150 // (below) pushes blocks into this queue instead of returning them to the |
160 // underlying allocator immediately. See MallocBlock::Deallocate for more | 151 // underlying allocator immediately. See MallocBlock::Deallocate for more |
161 // information. | 152 // information. |
162 // | 153 // |
163 // We can't use an STL class for this because we need to be careful not to | 154 // We can't use an STL class for this because we need to be careful not to |
164 // perform any heap de-allocations in any of the code in this class, since the | 155 // perform any heap de-allocations in any of the code in this class, since the |
165 // code in MallocBlock::Deallocate is not re-entrant. | 156 // code in MallocBlock::Deallocate is not re-entrant. |
166 template <typename QueueEntry> | 157 template <typename QueueEntry> |
167 class FreeQueue { | 158 class FreeQueue { |
168 public: | 159 public: |
169 FreeQueue() : q_front_(0), q_back_(0) {} | 160 FreeQueue() : q_front_(0), q_back_(0) {} |
170 | 161 |
171 bool Full() { | 162 bool Full() { |
172 return (q_front_ + 1) % kFreeQueueSize == q_back_; | 163 return (q_front_ + 1) % kFreeQueueSize == q_back_; |
173 } | 164 } |
174 | 165 |
175 void Push(const QueueEntry& block) { | 166 void Push(const QueueEntry& block) { |
176 q_[q_front_] = block; | 167 q_[q_front_] = block; |
177 q_front_ = (q_front_ + 1) % kFreeQueueSize; | 168 q_front_ = (q_front_ + 1) % kFreeQueueSize; |
178 } | 169 } |
179 | 170 |
180 QueueEntry Pop() { | 171 QueueEntry Pop() { |
181 RAW_CHECK(q_back_ != q_front_, "Queue is empty"); | |
182 const QueueEntry& ret = q_[q_back_]; | 172 const QueueEntry& ret = q_[q_back_]; |
183 q_back_ = (q_back_ + 1) % kFreeQueueSize; | 173 q_back_ = (q_back_ + 1) % kFreeQueueSize; |
184 return ret; | 174 return ret; |
185 } | 175 } |
186 | 176 |
187 size_t size() const { | 177 size_t size() const { |
188 return (q_front_ - q_back_ + kFreeQueueSize) % kFreeQueueSize; | 178 return (q_front_ - q_back_ + kFreeQueueSize) % kFreeQueueSize; |
189 } | 179 } |
190 | 180 |
191 private: | 181 private: |
192 // Maximum number of blocks kept in the free queue before being freed. | 182 // Maximum number of blocks kept in the free queue before being freed. |
193 static const int kFreeQueueSize = 1024; | 183 static const int kFreeQueueSize = 1024; |
194 | 184 |
195 QueueEntry q_[kFreeQueueSize]; | 185 QueueEntry q_[kFreeQueueSize]; |
196 int q_front_; | 186 int q_front_; |
197 int q_back_; | 187 int q_back_; |
198 }; | 188 }; |
199 | 189 |
200 struct MallocBlockQueueEntry { | 190 struct MallocBlockQueueEntry { |
201 MallocBlockQueueEntry() : block(NULL), size(0), | 191 MallocBlockQueueEntry() : block(NULL), size(0), |
202 num_deleter_pcs(0), deleter_threadid(0) {} | 192 num_deleter_pcs(0), deleter_threadid(0) {} |
203 MallocBlockQueueEntry(MallocBlock* b, size_t s) : block(b), size(s) { | 193 MallocBlockQueueEntry(MallocBlock* b, size_t s) : block(b), size(s) { |
204 if (FLAGS_max_free_queue_size != 0 && b != NULL) { | 194 if (FLAGS_max_free_queue_size != 0) { |
205 // Adjust the number of frames to skip (4) if you change the | 195 // Adjust the number of frames to skip (4) if you change the |
206 // location of this call. | 196 // location of this call. |
207 num_deleter_pcs = | 197 num_deleter_pcs = |
208 GetStackTrace(deleter_pcs, | 198 GetStackTrace(deleter_pcs, |
209 sizeof(deleter_pcs) / sizeof(deleter_pcs[0]), | 199 sizeof(deleter_pcs) / sizeof(deleter_pcs[0]), |
210 4); | 200 4); |
211 deleter_threadid = pthread_self(); | 201 deleter_threadid = pthread_self(); |
212 } else { | 202 } else { |
213 num_deleter_pcs = 0; | 203 num_deleter_pcs = 0; |
214 // Zero is an illegal pthread id by my reading of the pthread | 204 // Zero is an illegal pthread id by my reading of the pthread |
(...skipping 54 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
269 // start the program (see man malloc). | 259 // start the program (see man malloc). |
270 | 260 |
271 // We use either BASE_MALLOC or mmap to make the actual allocation. In | 261 // We use either BASE_MALLOC or mmap to make the actual allocation. In |
272 // order to remember which one of the two was used for any block, we store an | 262 // order to remember which one of the two was used for any block, we store an |
273 // appropriate magic word next to the block. | 263 // appropriate magic word next to the block. |
274 static const int kMagicMalloc = 0xDEADBEEF; | 264 static const int kMagicMalloc = 0xDEADBEEF; |
275 static const int kMagicMMap = 0xABCDEFAB; | 265 static const int kMagicMMap = 0xABCDEFAB; |
276 | 266 |
277 // This array will be filled with 0xCD, for use with memcmp. | 267 // This array will be filled with 0xCD, for use with memcmp. |
278 static unsigned char kMagicDeletedBuffer[1024]; | 268 static unsigned char kMagicDeletedBuffer[1024]; |
279 static pthread_once_t deleted_buffer_initialized_; | 269 static bool deleted_buffer_initialized_; |
280 static bool deleted_buffer_initialized_no_pthreads_; | |
281 | 270 |
282 private: // data layout | 271 private: // data layout |
283 | 272 |
284 // The four fields size1_,offset_,magic1_,alloc_type_ | 273 // The four fields size1_,offset_,magic1_,alloc_type_ |
285 // should together occupy a multiple of 16 bytes. (At the | 274 // should together occupy a multiple of 16 bytes. (At the |
286 // moment, sizeof(size_t) == 4 or 8 depending on piii vs | 275 // moment, sizeof(size_t) == 4 or 8 depending on piii vs |
287 // k8, and 4 of those sum to 16 or 32 bytes). | 276 // k8, and 4 of those sum to 16 or 32 bytes). |
288 // This, combined with BASE_MALLOC's alignment guarantees, | 277 // This, combined with BASE_MALLOC's alignment guarantees, |
289 // ensures that SSE types can be stored into the returned | 278 // ensures that SSE types can be stored into the returned |
290 // block, at &size2_. | 279 // block, at &size2_. |
(...skipping 274 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
565 } | 554 } |
566 } | 555 } |
567 | 556 |
568 static size_t FreeQueueSize() { | 557 static size_t FreeQueueSize() { |
569 SpinLockHolder l(&free_queue_lock_); | 558 SpinLockHolder l(&free_queue_lock_); |
570 return free_queue_size_; | 559 return free_queue_size_; |
571 } | 560 } |
572 | 561 |
573 static void ProcessFreeQueue(MallocBlock* b, size_t size, | 562 static void ProcessFreeQueue(MallocBlock* b, size_t size, |
574 int max_free_queue_size) { | 563 int max_free_queue_size) { |
575 // MallocBlockQueueEntry are about 144 in size, so we can only | 564 SpinLockHolder l(&free_queue_lock_); |
576 // use a small array of them on the stack. | |
577 MallocBlockQueueEntry entries[4]; | |
578 int num_entries = 0; | |
579 MallocBlockQueueEntry new_entry(b, size); | |
580 free_queue_lock_.Lock(); | |
581 if (free_queue_ == NULL) | 565 if (free_queue_ == NULL) |
582 free_queue_ = new FreeQueue<MallocBlockQueueEntry>; | 566 free_queue_ = new FreeQueue<MallocBlockQueueEntry>; |
583 RAW_CHECK(!free_queue_->Full(), "Free queue mustn't be full!"); | 567 RAW_CHECK(!free_queue_->Full(), "Free queue mustn't be full!"); |
584 | 568 |
585 if (b != NULL) { | 569 if (b != NULL) { |
586 free_queue_size_ += size + sizeof(MallocBlockQueueEntry); | 570 free_queue_size_ += size + sizeof(MallocBlockQueueEntry); |
| 571 MallocBlockQueueEntry new_entry(b, size); |
587 free_queue_->Push(new_entry); | 572 free_queue_->Push(new_entry); |
588 } | 573 } |
589 | 574 |
590 // Free blocks until the total size of unfreed blocks no longer exceeds | 575 // Free blocks until the total size of unfreed blocks no longer exceeds |
591 // max_free_queue_size, and the free queue has at least one free | 576 // max_free_queue_size, and the free queue has at least one free |
592 // space in it. | 577 // space in it. |
593 while (free_queue_size_ > max_free_queue_size || free_queue_->Full()) { | 578 while (free_queue_size_ > max_free_queue_size || free_queue_->Full()) { |
594 RAW_CHECK(num_entries < arraysize(entries), "entries array overflow"); | 579 MallocBlockQueueEntry cur = free_queue_->Pop(); |
595 entries[num_entries] = free_queue_->Pop(); | 580 CheckForDanglingWrites(cur); |
596 free_queue_size_ -= | 581 free_queue_size_ -= cur.size + sizeof(MallocBlockQueueEntry); |
597 entries[num_entries].size + sizeof(MallocBlockQueueEntry); | 582 BASE_FREE(cur.block); |
598 num_entries++; | |
599 if (num_entries == arraysize(entries)) { | |
600 // The queue will not be full at this point, so it is ok to | |
601 // release the lock. The queue may still contain more than | |
602 // max_free_queue_size, but this is not a strict invariant. | |
603 free_queue_lock_.Unlock(); | |
604 for (int i = 0; i < num_entries; i++) { | |
605 CheckForDanglingWrites(entries[i]); | |
606 BASE_FREE(entries[i].block); | |
607 } | |
608 num_entries = 0; | |
609 free_queue_lock_.Lock(); | |
610 } | |
611 } | 583 } |
612 RAW_CHECK(free_queue_size_ >= 0, "Free queue size went negative!"); | 584 RAW_CHECK(free_queue_size_ >= 0, "Free queue size went negative!"); |
613 free_queue_lock_.Unlock(); | |
614 for (int i = 0; i < num_entries; i++) { | |
615 CheckForDanglingWrites(entries[i]); | |
616 BASE_FREE(entries[i].block); | |
617 } | |
618 } | |
619 | |
620 static void InitDeletedBuffer() { | |
621 memset(kMagicDeletedBuffer, kMagicDeletedByte, sizeof(kMagicDeletedBuffer)); | |
622 deleted_buffer_initialized_no_pthreads_ = true; | |
623 } | 585 } |
624 | 586 |
625 static void CheckForDanglingWrites(const MallocBlockQueueEntry& queue_entry) { | 587 static void CheckForDanglingWrites(const MallocBlockQueueEntry& queue_entry) { |
626 // Initialize the buffer if necessary. | 588 // Initialize the buffer if necessary. |
627 if (pthread_once) | 589 if (!deleted_buffer_initialized_) { |
628 pthread_once(&deleted_buffer_initialized_, &InitDeletedBuffer); | 590 // This is threadsafe. We hold free_queue_lock_. |
629 if (!deleted_buffer_initialized_no_pthreads_) { | 591 memset(kMagicDeletedBuffer, 0xcd, sizeof(kMagicDeletedBuffer)); |
630 // This will be the case on systems that don't link in pthreads, | 592 deleted_buffer_initialized_ = true; |
631 // including on FreeBSD where pthread_once has a non-zero address | |
632 // (but doesn't do anything) even when pthreads isn't linked in. | |
633 InitDeletedBuffer(); | |
634 } | 593 } |
635 | 594 |
636 const unsigned char* p = | 595 const unsigned char* p = |
637 reinterpret_cast<unsigned char*>(queue_entry.block); | 596 reinterpret_cast<unsigned char*>(queue_entry.block); |
638 | 597 |
639 static const size_t size_of_buffer = sizeof(kMagicDeletedBuffer); | 598 static const size_t size_of_buffer = sizeof(kMagicDeletedBuffer); |
640 const size_t size = queue_entry.size; | 599 const size_t size = queue_entry.size; |
641 const size_t buffers = size / size_of_buffer; | 600 const size_t buffers = size / size_of_buffer; |
642 const size_t remainder = size % size_of_buffer; | 601 const size_t remainder = size % size_of_buffer; |
643 size_t buffer_idx; | 602 size_t buffer_idx; |
(...skipping 15 matching lines...) Expand all Loading... |
659 RAW_LOG(ERROR, | 618 RAW_LOG(ERROR, |
660 "Found a corrupted memory buffer in MallocBlock (may be offset " | 619 "Found a corrupted memory buffer in MallocBlock (may be offset " |
661 "from user ptr): buffer index: %zd, buffer ptr: %p, size of " | 620 "from user ptr): buffer index: %zd, buffer ptr: %p, size of " |
662 "buffer: %zd", buffer_idx, buffer, size_of_buffer); | 621 "buffer: %zd", buffer_idx, buffer, size_of_buffer); |
663 | 622 |
664 // The magic deleted buffer should only be 1024 bytes, but in case | 623 // The magic deleted buffer should only be 1024 bytes, but in case |
665 // this changes, let's put an upper limit on the number of debug | 624 // this changes, let's put an upper limit on the number of debug |
666 // lines we'll output: | 625 // lines we'll output: |
667 if (size_of_buffer <= 1024) { | 626 if (size_of_buffer <= 1024) { |
668 for (int i = 0; i < size_of_buffer; ++i) { | 627 for (int i = 0; i < size_of_buffer; ++i) { |
669 if (buffer[i] != kMagicDeletedByte) { | 628 if (buffer[i] != 0xcd) { |
670 RAW_LOG(ERROR, "Buffer byte %d is 0x%02x (should be 0x%02x).", | 629 RAW_LOG(ERROR, "Buffer byte %d is 0x%02x (should be 0xcd).", |
671 i, buffer[i], kMagicDeletedByte); | 630 i, buffer[i]); |
672 } | 631 } |
673 } | 632 } |
674 } else { | 633 } else { |
675 RAW_LOG(ERROR, "Buffer too large to print corruption."); | 634 RAW_LOG(ERROR, "Buffer too large to print corruption."); |
676 } | 635 } |
677 | 636 |
678 const MallocBlock* b = queue_entry.block; | 637 const MallocBlock* b = queue_entry.block; |
679 const size_t size = queue_entry.size; | 638 const size_t size = queue_entry.size; |
680 if (queue_entry.num_deleter_pcs > 0) { | 639 if (queue_entry.num_deleter_pcs > 0) { |
681 TracePrintf(STDERR_FILENO, "Deleted by thread %p\n", | 640 TracePrintf(STDERR_FILENO, "Deleted by thread %p\n", |
(...skipping 23 matching lines...) Expand all Loading... |
705 RAW_LOG(ERROR, | 664 RAW_LOG(ERROR, |
706 "Skipping the printing of the deleter's stack! Its stack was " | 665 "Skipping the printing of the deleter's stack! Its stack was " |
707 "not found; either the corruption occurred too early in " | 666 "not found; either the corruption occurred too early in " |
708 "execution to obtain a stack trace or --max_free_queue_size was " | 667 "execution to obtain a stack trace or --max_free_queue_size was " |
709 "set to 0."); | 668 "set to 0."); |
710 } | 669 } |
711 | 670 |
712 RAW_LOG(FATAL, | 671 RAW_LOG(FATAL, |
713 "Memory was written to after being freed. MallocBlock: %p, user " | 672 "Memory was written to after being freed. MallocBlock: %p, user " |
714 "ptr: %p, size: %zd. If you can't find the source of the error, " | 673 "ptr: %p, size: %zd. If you can't find the source of the error, " |
715 "try using ASan (http://code.google.com/p/address-sanitizer/), " | 674 "try using valgrind or purify, or study the output of the " |
716 "Valgrind, or Purify, or study the " | 675 "deleter's stack printed above.", b, b->data_addr(), size); |
717 "output of the deleter's stack printed above.", | |
718 b, b->data_addr(), size); | |
719 } | 676 } |
720 | 677 |
721 static MallocBlock* FromRawPointer(void* p) { | 678 static MallocBlock* FromRawPointer(void* p) { |
722 const size_t data_offset = MallocBlock::data_offset(); | 679 const size_t data_offset = MallocBlock::data_offset(); |
723 // Find the header just before client's memory. | 680 // Find the header just before client's memory. |
724 MallocBlock *mb = reinterpret_cast<MallocBlock *>( | 681 MallocBlock *mb = reinterpret_cast<MallocBlock *>( |
725 reinterpret_cast<char *>(p) - data_offset); | 682 reinterpret_cast<char *>(p) - data_offset); |
726 // If mb->alloc_type_ is kMagicDeletedSizeT, we're not an ok pointer. | 683 // If mb->alloc_type_ is kMagicDeletedSizeT, we're not an ok pointer. |
727 if (mb->alloc_type_ == kMagicDeletedSizeT) { | 684 if (mb->alloc_type_ == kMagicDeletedSizeT) { |
728 RAW_LOG(FATAL, "memory allocation bug: object at %p has been already" | 685 RAW_LOG(FATAL, "memory allocation bug: object at %p has been already" |
729 " deallocated; or else a word before the object has been" | 686 " deallocated; or else a word before the object has been" |
730 " corrupted (memory stomping bug)", p); | 687 " corrupted (memory stomping bug)", p); |
731 } | 688 } |
732 // If mb->offset_ is zero (common case), mb is the real header. If | 689 // If mb->offset_ is zero (common case), mb is the real header. If |
733 // mb->offset_ is non-zero, this block was allocated by memalign, and | 690 // mb->offset_ is non-zero, this block was allocated by memalign, and |
734 // mb->offset_ is the distance backwards to the real header from mb, | 691 // mb->offset_ is the distance backwards to the real header from mb, |
735 // which is a fake header. The following subtraction works for both zero | 692 // which is a fake header. The following subtraction works for both zero |
736 // and non-zero values. | 693 // and non-zero values. |
737 return reinterpret_cast<MallocBlock *>( | 694 return reinterpret_cast<MallocBlock *>( |
738 reinterpret_cast<char *>(mb) - mb->offset_); | 695 reinterpret_cast<char *>(mb) - mb->offset_); |
739 } | 696 } |
740 static const MallocBlock* FromRawPointer(const void* p) { | 697 static const MallocBlock* FromRawPointer(const void* p) { |
741 // const-safe version: we just cast about | 698 // const-safe version: we just cast about |
742 return FromRawPointer(const_cast<void*>(p)); | 699 return FromRawPointer(const_cast<void*>(p)); |
743 } | 700 } |
744 | 701 |
745 void Check(int type) const { | 702 void Check(int type) { |
746 alloc_map_lock_.Lock(); | 703 alloc_map_lock_.Lock(); |
747 CheckLocked(type); | 704 CheckLocked(type); |
748 alloc_map_lock_.Unlock(); | 705 alloc_map_lock_.Unlock(); |
749 } | 706 } |
750 | 707 |
751 static bool CheckEverything() { | 708 static bool CheckEverything() { |
752 alloc_map_lock_.Lock(); | 709 alloc_map_lock_.Lock(); |
753 if (alloc_map_ != NULL) alloc_map_->Iterate(CheckCallback, 0); | 710 if (alloc_map_ != NULL) alloc_map_->Iterate(CheckCallback, 0); |
754 alloc_map_lock_.Unlock(); | 711 alloc_map_lock_.Unlock(); |
755 return true; // if we get here, we're okay | 712 return true; // if we get here, we're okay |
(...skipping 57 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
813 const int MallocBlock::kMagicMMap; | 770 const int MallocBlock::kMagicMMap; |
814 | 771 |
815 MallocBlock::AllocMap* MallocBlock::alloc_map_ = NULL; | 772 MallocBlock::AllocMap* MallocBlock::alloc_map_ = NULL; |
816 SpinLock MallocBlock::alloc_map_lock_(SpinLock::LINKER_INITIALIZED); | 773 SpinLock MallocBlock::alloc_map_lock_(SpinLock::LINKER_INITIALIZED); |
817 | 774 |
818 FreeQueue<MallocBlockQueueEntry>* MallocBlock::free_queue_ = NULL; | 775 FreeQueue<MallocBlockQueueEntry>* MallocBlock::free_queue_ = NULL; |
819 size_t MallocBlock::free_queue_size_ = 0; | 776 size_t MallocBlock::free_queue_size_ = 0; |
820 SpinLock MallocBlock::free_queue_lock_(SpinLock::LINKER_INITIALIZED); | 777 SpinLock MallocBlock::free_queue_lock_(SpinLock::LINKER_INITIALIZED); |
821 | 778 |
822 unsigned char MallocBlock::kMagicDeletedBuffer[1024]; | 779 unsigned char MallocBlock::kMagicDeletedBuffer[1024]; |
823 pthread_once_t MallocBlock::deleted_buffer_initialized_ = PTHREAD_ONCE_INIT; | 780 bool MallocBlock::deleted_buffer_initialized_ = false; |
824 bool MallocBlock::deleted_buffer_initialized_no_pthreads_ = false; | |
825 | 781 |
826 const char* const MallocBlock::kAllocName[] = { | 782 const char* const MallocBlock::kAllocName[] = { |
827 "malloc", | 783 "malloc", |
828 "new", | 784 "new", |
829 "new []", | 785 "new []", |
830 NULL, | 786 NULL, |
831 }; | 787 }; |
832 | 788 |
833 const char* const MallocBlock::kDeallocName[] = { | 789 const char* const MallocBlock::kDeallocName[] = { |
834 "free", | 790 "free", |
(...skipping 173 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1008 if (result && (strcmp(name, "generic.current_allocated_bytes") == 0)) { | 964 if (result && (strcmp(name, "generic.current_allocated_bytes") == 0)) { |
1009 // Subtract bytes kept in the free queue | 965 // Subtract bytes kept in the free queue |
1010 size_t qsize = MallocBlock::FreeQueueSize(); | 966 size_t qsize = MallocBlock::FreeQueueSize(); |
1011 if (*value >= qsize) { | 967 if (*value >= qsize) { |
1012 *value -= qsize; | 968 *value -= qsize; |
1013 } | 969 } |
1014 } | 970 } |
1015 return result; | 971 return result; |
1016 } | 972 } |
1017 | 973 |
1018 virtual bool VerifyNewMemory(const void* p) { | 974 virtual bool VerifyNewMemory(void* p) { |
1019 if (p) MallocBlock::FromRawPointer(p)->Check(MallocBlock::kNewType); | 975 if (p) MallocBlock::FromRawPointer(p)->Check(MallocBlock::kNewType); |
1020 return true; | 976 return true; |
1021 } | 977 } |
1022 | 978 |
1023 virtual bool VerifyArrayNewMemory(const void* p) { | 979 virtual bool VerifyArrayNewMemory(void* p) { |
1024 if (p) MallocBlock::FromRawPointer(p)->Check(MallocBlock::kArrayNewType); | 980 if (p) MallocBlock::FromRawPointer(p)->Check(MallocBlock::kArrayNewType); |
1025 return true; | 981 return true; |
1026 } | 982 } |
1027 | 983 |
1028 virtual bool VerifyMallocMemory(const void* p) { | 984 virtual bool VerifyMallocMemory(void* p) { |
1029 if (p) MallocBlock::FromRawPointer(p)->Check(MallocBlock::kMallocType); | 985 if (p) MallocBlock::FromRawPointer(p)->Check(MallocBlock::kMallocType); |
1030 return true; | 986 return true; |
1031 } | 987 } |
1032 | 988 |
1033 virtual bool VerifyAllMemory() { | 989 virtual bool VerifyAllMemory() { |
1034 return MallocBlock::CheckEverything(); | 990 return MallocBlock::CheckEverything(); |
1035 } | 991 } |
1036 | 992 |
1037 virtual bool MallocMemoryStats(int* blocks, size_t* total, | 993 virtual bool MallocMemoryStats(int* blocks, size_t* total, |
1038 int histogram[kMallocHistogramSize]) { | 994 int histogram[kMallocHistogramSize]) { |
1039 return MallocBlock::MemoryStats(blocks, total, histogram); | 995 return MallocBlock::MemoryStats(blocks, total, histogram); |
1040 } | 996 } |
1041 | 997 |
| 998 virtual size_t GetAllocatedSize(void* p) { |
| 999 if (p) { |
| 1000 return MallocBlock::FromRawPointer(p)->data_size(); |
| 1001 } |
| 1002 return 0; |
| 1003 } |
1042 virtual size_t GetEstimatedAllocatedSize(size_t size) { | 1004 virtual size_t GetEstimatedAllocatedSize(size_t size) { |
1043 return size; | 1005 return size; |
1044 } | 1006 } |
1045 | 1007 |
1046 virtual size_t GetAllocatedSize(const void* p) { | |
1047 if (p) { | |
1048 RAW_CHECK(GetOwnership(p) != MallocExtension::kNotOwned, | |
1049 "ptr not allocated by tcmalloc"); | |
1050 return MallocBlock::FromRawPointer(p)->data_size(); | |
1051 } | |
1052 return 0; | |
1053 } | |
1054 | |
1055 virtual MallocExtension::Ownership GetOwnership(const void* p) { | |
1056 if (p) { | |
1057 const MallocBlock* mb = MallocBlock::FromRawPointer(p); | |
1058 return TCMallocImplementation::GetOwnership(mb); | |
1059 } | |
1060 return MallocExtension::kNotOwned; // nobody owns NULL | |
1061 } | |
1062 | |
1063 virtual void GetFreeListSizes(vector<MallocExtension::FreeListInfo>* v) { | 1008 virtual void GetFreeListSizes(vector<MallocExtension::FreeListInfo>* v) { |
1064 static const char* kDebugFreeQueue = "debug.free_queue"; | 1009 static const char* kDebugFreeQueue = "debug.free_queue"; |
1065 | 1010 |
1066 TCMallocImplementation::GetFreeListSizes(v); | 1011 TCMallocImplementation::GetFreeListSizes(v); |
1067 | 1012 |
1068 MallocExtension::FreeListInfo i; | 1013 MallocExtension::FreeListInfo i; |
1069 i.type = kDebugFreeQueue; | 1014 i.type = kDebugFreeQueue; |
1070 i.min_object_size = 0; | 1015 i.min_object_size = 0; |
1071 i.max_object_size = numeric_limits<size_t>::max(); | 1016 i.max_object_size = numeric_limits<size_t>::max(); |
1072 i.total_bytes_free = MallocBlock::FreeQueueSize(); | 1017 i.total_bytes_free = MallocBlock::FreeQueueSize(); |
1073 v->push_back(i); | 1018 v->push_back(i); |
1074 } | 1019 } |
1075 | 1020 |
1076 }; | 1021 }; |
1077 | 1022 |
1078 static DebugMallocImplementation debug_malloc_implementation; | 1023 static DebugMallocImplementation debug_malloc_implementation; |
1079 | 1024 |
1080 REGISTER_MODULE_INITIALIZER(debugallocation, { | 1025 REGISTER_MODULE_INITIALIZER(debugallocation, { |
1081 // Either we or valgrind will control memory management. We | 1026 // Either we or valgrind will control memory management. We |
1082 // register our extension if we're the winner. Otherwise let | 1027 // register our extension if we're the winner. |
1083 // Valgrind use its own malloc (so don't register our extension). | 1028 if (RunningOnValgrind()) { |
1084 if (!RunningOnValgrind()) { | 1029 // Let Valgrind uses its own malloc (so don't register our extension). |
| 1030 } else { |
1085 MallocExtension::Register(&debug_malloc_implementation); | 1031 MallocExtension::Register(&debug_malloc_implementation); |
1086 } | |
1087 }); | |
1088 | |
1089 REGISTER_MODULE_DESTRUCTOR(debugallocation, { | |
1090 if (!RunningOnValgrind()) { | |
1091 // When the program exits, check all blocks still in the free | 1032 // When the program exits, check all blocks still in the free |
1092 // queue for corruption. | 1033 // queue for corruption. |
1093 DanglingWriteChecker(); | 1034 atexit(DanglingWriteChecker); |
1094 } | 1035 } |
1095 }); | 1036 }); |
1096 | 1037 |
1097 // ========================================================================= // | 1038 // ========================================================================= // |
1098 | 1039 |
1099 // This is mostly the same a cpp_alloc in tcmalloc.cc. | 1040 // This is mostly the same a cpp_alloc in tcmalloc.cc. |
1100 // TODO(csilvers): change Allocate() above to call cpp_alloc, so we | 1041 // TODO(csilvers): write a wrapper for new-handler so we don't have to |
1101 // don't have to reproduce the logic here. To make tc_new_mode work | 1042 // copy this code so much. |
1102 // properly, I think we'll need to separate out the logic of throwing | |
1103 // from the logic of calling the new-handler. | |
1104 inline void* debug_cpp_alloc(size_t size, int new_type, bool nothrow) { | 1043 inline void* debug_cpp_alloc(size_t size, int new_type, bool nothrow) { |
1105 for (;;) { | 1044 for (;;) { |
1106 void* p = DebugAllocate(size, new_type); | 1045 void* p = DebugAllocate(size, new_type); |
1107 #ifdef PREANSINEW | 1046 #ifdef PREANSINEW |
1108 return p; | 1047 return p; |
1109 #else | 1048 #else |
1110 if (p == NULL) { // allocation failed | 1049 if (p == NULL) { // allocation failed |
1111 // Get the current new handler. NB: this function is not | 1050 // Get the current new handler. NB: this function is not |
1112 // thread-safe. We make a feeble stab at making it so here, but | 1051 // thread-safe. We make a feeble stab at making it so here, but |
1113 // this lock only protects against tcmalloc interfering with | 1052 // this lock only protects against tcmalloc interfering with |
(...skipping 295 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1409 return BASE_MALLOPT(cmd, value); | 1348 return BASE_MALLOPT(cmd, value); |
1410 } | 1349 } |
1411 | 1350 |
1412 #ifdef HAVE_STRUCT_MALLINFO | 1351 #ifdef HAVE_STRUCT_MALLINFO |
1413 extern "C" PERFTOOLS_DLL_DECL struct mallinfo tc_mallinfo(void) __THROW { | 1352 extern "C" PERFTOOLS_DLL_DECL struct mallinfo tc_mallinfo(void) __THROW { |
1414 return BASE_MALLINFO(); | 1353 return BASE_MALLINFO(); |
1415 } | 1354 } |
1416 #endif | 1355 #endif |
1417 | 1356 |
1418 extern "C" PERFTOOLS_DLL_DECL size_t tc_malloc_size(void* ptr) __THROW { | 1357 extern "C" PERFTOOLS_DLL_DECL size_t tc_malloc_size(void* ptr) __THROW { |
1419 return MallocExtension::instance()->GetAllocatedSize(ptr); | 1358 if (!ptr) { |
| 1359 return 0; |
| 1360 } |
| 1361 MallocBlock* mb = MallocBlock::FromRawPointer(ptr); |
| 1362 // This is just to make sure we actually own mb (and ptr). We don't |
| 1363 // use the actual value, just the 'exception' it raises on error. |
| 1364 (void)BASE_MALLOC_SIZE(mb); |
| 1365 return mb->data_size(); |
1420 } | 1366 } |
| 1367 |
| 1368 // Override __libc_memalign in libc on linux boxes. |
| 1369 // They have a bug in libc that causes them (very rarely) to allocate |
| 1370 // with __libc_memalign() yet deallocate with free(). |
| 1371 // This function is an exception to the rule of calling MallocHook method |
| 1372 // from the stack frame of the allocation function; |
| 1373 // heap-checker handles this special case explicitly. |
| 1374 static void *MemalignOverride(size_t align, size_t size, const void *caller) |
| 1375 __THROW ATTRIBUTE_SECTION(google_malloc); |
| 1376 |
| 1377 static void *MemalignOverride(size_t align, size_t size, const void *caller) |
| 1378 __THROW { |
| 1379 void *p = do_debug_memalign_or_debug_cpp_memalign(align, size); |
| 1380 MallocHook::InvokeNewHook(p, size); |
| 1381 return p; |
| 1382 } |
| 1383 void *(*__memalign_hook)(size_t, size_t, const void *) = MemalignOverride; |
OLD | NEW |