OLD | NEW |
1 // Copyright (c) 2005, Google Inc. | 1 // Copyright (c) 2005, Google Inc. |
2 // All rights reserved. | 2 // All rights reserved. |
3 // | 3 // |
4 // Redistribution and use in source and binary forms, with or without | 4 // Redistribution and use in source and binary forms, with or without |
5 // modification, are permitted provided that the following conditions are | 5 // modification, are permitted provided that the following conditions are |
6 // met: | 6 // met: |
7 // | 7 // |
8 // * Redistributions of source code must retain the above copyright | 8 // * Redistributions of source code must retain the above copyright |
9 // notice, this list of conditions and the following disclaimer. | 9 // notice, this list of conditions and the following disclaimer. |
10 // * Redistributions in binary form must reproduce the above | 10 // * Redistributions in binary form must reproduce the above |
(...skipping 69 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
80 // TODO: Bias reclamation to larger addresses | 80 // TODO: Bias reclamation to larger addresses |
81 // TODO: implement mallinfo/mallopt | 81 // TODO: implement mallinfo/mallopt |
82 // TODO: Better testing | 82 // TODO: Better testing |
83 // | 83 // |
84 // 9/28/2003 (new page-level allocator replaces ptmalloc2): | 84 // 9/28/2003 (new page-level allocator replaces ptmalloc2): |
85 // * malloc/free of small objects goes from ~300 ns to ~50 ns. | 85 // * malloc/free of small objects goes from ~300 ns to ~50 ns. |
86 // * allocation of a reasonably complicated struct | 86 // * allocation of a reasonably complicated struct |
87 // goes from about 1100 ns to about 300 ns. | 87 // goes from about 1100 ns to about 300 ns. |
88 | 88 |
89 #include "config.h" | 89 #include "config.h" |
90 #include <google/tcmalloc.h> | 90 #include <gperftools/tcmalloc.h> |
91 | 91 |
92 #include <errno.h> // for ENOMEM, EINVAL, errno | 92 #include <errno.h> // for ENOMEM, EINVAL, errno |
93 #ifdef HAVE_SYS_CDEFS_H | 93 #ifdef HAVE_SYS_CDEFS_H |
94 #include <sys/cdefs.h> // for __THROW | 94 #include <sys/cdefs.h> // for __THROW |
95 #endif | 95 #endif |
96 #ifdef HAVE_FEATURES_H | |
97 #include <features.h> // for __GLIBC__ | |
98 #endif | |
99 #if defined HAVE_STDINT_H | 96 #if defined HAVE_STDINT_H |
100 #include <stdint.h> | 97 #include <stdint.h> |
101 #elif defined HAVE_INTTYPES_H | 98 #elif defined HAVE_INTTYPES_H |
102 #include <inttypes.h> | 99 #include <inttypes.h> |
103 #else | 100 #else |
104 #include <sys/types.h> | 101 #include <sys/types.h> |
105 #endif | 102 #endif |
106 #include <stddef.h> // for size_t, NULL | 103 #include <stddef.h> // for size_t, NULL |
107 #include <stdlib.h> // for getenv | 104 #include <stdlib.h> // for getenv |
108 #include <string.h> // for strcmp, memset, strlen, etc | 105 #include <string.h> // for strcmp, memset, strlen, etc |
109 #ifdef HAVE_UNISTD_H | 106 #ifdef HAVE_UNISTD_H |
110 #include <unistd.h> // for getpagesize, write, etc | 107 #include <unistd.h> // for getpagesize, write, etc |
111 #endif | 108 #endif |
112 #include <algorithm> // for max, min | 109 #include <algorithm> // for max, min |
113 #include <limits> // for numeric_limits | 110 #include <limits> // for numeric_limits |
114 #include <new> // for nothrow_t (ptr only), etc | 111 #include <new> // for nothrow_t (ptr only), etc |
115 #include <vector> // for vector | 112 #include <vector> // for vector |
116 | 113 |
117 #include <google/malloc_extension.h> | 114 #include <gperftools/malloc_extension.h> |
118 #include <google/malloc_hook.h> // for MallocHook | 115 #include <gperftools/malloc_hook.h> // for MallocHook |
119 #include "base/basictypes.h" // for int64 | 116 #include "base/basictypes.h" // for int64 |
120 #include "base/commandlineflags.h" // for RegisterFlagValidator, etc | 117 #include "base/commandlineflags.h" // for RegisterFlagValidator, etc |
121 #include "base/dynamic_annotations.h" // for RunningOnValgrind | 118 #include "base/dynamic_annotations.h" // for RunningOnValgrind |
122 #include "base/spinlock.h" // for SpinLockHolder | 119 #include "base/spinlock.h" // for SpinLockHolder |
123 #include "central_freelist.h" // for CentralFreeListPadded | 120 #include "central_freelist.h" // for CentralFreeListPadded |
124 #include "common.h" // for StackTrace, kPageShift, etc | 121 #include "common.h" // for StackTrace, kPageShift, etc |
125 #include "free_list.h" // for FL_Init | 122 #include "free_list.h" // for FL_Init |
126 #include "internal_logging.h" // for ASSERT, TCMalloc_Printer, etc | 123 #include "internal_logging.h" // for ASSERT, TCMalloc_Printer, etc |
127 #include "malloc_hook-inl.h" // for MallocHook::InvokeNewHook, etc | 124 #include "malloc_hook-inl.h" // for MallocHook::InvokeNewHook, etc |
128 #include "page_heap.h" // for PageHeap, PageHeap::Stats | 125 #include "page_heap.h" // for PageHeap, PageHeap::Stats |
(...skipping 14 matching lines...) Expand all Loading... |
143 # include <sys/malloc.h> | 140 # include <sys/malloc.h> |
144 # elif defined(HAVE_MALLOC_MALLOC_H) | 141 # elif defined(HAVE_MALLOC_MALLOC_H) |
145 # include <malloc/malloc.h> | 142 # include <malloc/malloc.h> |
146 # endif | 143 # endif |
147 #endif | 144 #endif |
148 | 145 |
149 #if (defined(_WIN32) && !defined(__CYGWIN__) && !defined(__CYGWIN32__)) && !defi
ned(WIN32_OVERRIDE_ALLOCATORS) | 146 #if (defined(_WIN32) && !defined(__CYGWIN__) && !defined(__CYGWIN32__)) && !defi
ned(WIN32_OVERRIDE_ALLOCATORS) |
150 # define WIN32_DO_PATCHING 1 | 147 # define WIN32_DO_PATCHING 1 |
151 #endif | 148 #endif |
152 | 149 |
153 // GLibc 2.14+ requires the hook functions be declared volatile, based on the | 150 // Some windows file somewhere (at least on cygwin) #define's small (!) |
154 // value of the define __MALLOC_HOOK_VOLATILE. For compatibility with | 151 // For instance, <windows.h> appears to have "#define small char". |
155 // older/non-GLibc implementations, provide an empty definition. | 152 #undef small |
156 #if !defined(__MALLOC_HOOK_VOLATILE) | |
157 #define __MALLOC_HOOK_VOLATILE | |
158 #endif | |
159 | 153 |
160 using STL_NAMESPACE::max; | 154 using STL_NAMESPACE::max; |
161 using STL_NAMESPACE::numeric_limits; | 155 using STL_NAMESPACE::numeric_limits; |
162 using STL_NAMESPACE::vector; | 156 using STL_NAMESPACE::vector; |
| 157 |
| 158 #include "libc_override.h" |
| 159 |
| 160 // __THROW is defined in glibc (via <sys/cdefs.h>). It means, |
| 161 // counter-intuitively, "This function will never throw an exception." |
| 162 // It's an optional optimization tool, but we may need to use it to |
| 163 // match glibc prototypes. |
| 164 #ifndef __THROW // I guess we're not on a glibc system |
| 165 # define __THROW // __THROW is just an optimization, so ok to make it "" |
| 166 #endif |
| 167 |
163 using tcmalloc::AlignmentForSize; | 168 using tcmalloc::AlignmentForSize; |
| 169 using tcmalloc::kLog; |
| 170 using tcmalloc::kCrash; |
| 171 using tcmalloc::kCrashWithStats; |
| 172 using tcmalloc::Log; |
164 using tcmalloc::PageHeap; | 173 using tcmalloc::PageHeap; |
165 using tcmalloc::PageHeapAllocator; | 174 using tcmalloc::PageHeapAllocator; |
166 using tcmalloc::SizeMap; | 175 using tcmalloc::SizeMap; |
167 using tcmalloc::Span; | 176 using tcmalloc::Span; |
168 using tcmalloc::StackTrace; | 177 using tcmalloc::StackTrace; |
169 using tcmalloc::Static; | 178 using tcmalloc::Static; |
170 using tcmalloc::ThreadCache; | 179 using tcmalloc::ThreadCache; |
171 | 180 |
172 // __THROW is defined in glibc systems. It means, counter-intuitively, | |
173 // "This function will never throw an exception." It's an optional | |
174 // optimization tool, but we may need to use it to match glibc prototypes. | |
175 #ifndef __THROW // I guess we're not on a glibc system | |
176 # define __THROW // __THROW is just an optimization, so ok to make it "" | |
177 #endif | |
178 | |
179 // ---- Double free debug declarations | 181 // ---- Double free debug declarations |
180 static size_t ExcludeSpaceForMark(size_t size); | 182 static size_t ExcludeSpaceForMark(size_t size); |
181 static void AddRoomForMark(size_t* size); | 183 static void AddRoomForMark(size_t* size); |
182 static void ExcludeMarkFromSize(size_t* new_size); | 184 static void ExcludeMarkFromSize(size_t* new_size); |
183 static void MarkAllocatedRegion(void* ptr); | 185 static void MarkAllocatedRegion(void* ptr); |
184 static void ValidateAllocatedRegion(void* ptr, size_t cl); | 186 static void ValidateAllocatedRegion(void* ptr, size_t cl); |
185 // ---- End Double free debug declarations | 187 // ---- End Double free debug declarations |
186 | 188 |
187 DECLARE_int64(tcmalloc_sample_parameter); | 189 DECLARE_int64(tcmalloc_sample_parameter); |
188 DECLARE_double(tcmalloc_release_rate); | 190 DECLARE_double(tcmalloc_release_rate); |
(...skipping 84 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
273 // Some non-standard extensions that we support. | 275 // Some non-standard extensions that we support. |
274 | 276 |
275 // This is equivalent to | 277 // This is equivalent to |
276 // OS X: malloc_size() | 278 // OS X: malloc_size() |
277 // glibc: malloc_usable_size() | 279 // glibc: malloc_usable_size() |
278 // Windows: _msize() | 280 // Windows: _msize() |
279 size_t tc_malloc_size(void* p) __THROW | 281 size_t tc_malloc_size(void* p) __THROW |
280 ATTRIBUTE_SECTION(google_malloc); | 282 ATTRIBUTE_SECTION(google_malloc); |
281 } // extern "C" | 283 } // extern "C" |
282 | 284 |
283 // Override the libc functions to prefer our own instead. This comes | |
284 // first so code in tcmalloc.cc can use the overridden versions. One | |
285 // exception: in windows, by default, we patch our code into these | |
286 // functions (via src/windows/patch_function.cc) rather than override | |
287 // them. In that case, we don't want to do this overriding here. | |
288 #if !defined(WIN32_DO_PATCHING) | |
289 | |
290 // TODO(mbelshe): Turn off TCMalloc's symbols for libc. We do that | |
291 // elsewhere. | |
292 #ifndef _WIN32 | |
293 | |
294 #if defined(__GNUC__) && !defined(__MACH__) | |
295 // Potentially faster variants that use the gcc alias extension. | |
296 // FreeBSD does support aliases, but apparently not correctly. :-( | |
297 // NOTE: we make many of these symbols weak, but do so in the makefile | |
298 // (via objcopy -W) and not here. That ends up being more portable. | |
299 # define ALIAS(x) __attribute__ ((alias (x))) | |
300 void* operator new(size_t size) throw (std::bad_alloc) ALIAS("tc_new"); | |
301 void operator delete(void* p) __THROW ALIAS("tc_delete"); | |
302 void* operator new[](size_t size) throw (std::bad_alloc) ALIAS("tc_newarray"); | |
303 void operator delete[](void* p) __THROW ALIAS("tc_deletearray"); | |
304 void* operator new(size_t size, const std::nothrow_t&) __THROW | |
305 ALIAS("tc_new_nothrow"); | |
306 void* operator new[](size_t size, const std::nothrow_t&) __THROW | |
307 ALIAS("tc_newarray_nothrow"); | |
308 void operator delete(void* size, const std::nothrow_t&) __THROW | |
309 ALIAS("tc_delete_nothrow"); | |
310 void operator delete[](void* size, const std::nothrow_t&) __THROW | |
311 ALIAS("tc_deletearray_nothrow"); | |
312 extern "C" { | |
313 void* malloc(size_t size) __THROW ALIAS("tc_malloc"); | |
314 void free(void* ptr) __THROW ALIAS("tc_free"); | |
315 void* realloc(void* ptr, size_t size) __THROW ALIAS("tc_realloc"); | |
316 void* calloc(size_t n, size_t size) __THROW ALIAS("tc_calloc"); | |
317 void cfree(void* ptr) __THROW ALIAS("tc_cfree"); | |
318 void* memalign(size_t align, size_t s) __THROW ALIAS("tc_memalign"); | |
319 void* valloc(size_t size) __THROW ALIAS("tc_valloc"); | |
320 void* pvalloc(size_t size) __THROW ALIAS("tc_pvalloc"); | |
321 int posix_memalign(void** r, size_t a, size_t s) __THROW | |
322 ALIAS("tc_posix_memalign"); | |
323 void malloc_stats(void) __THROW ALIAS("tc_malloc_stats"); | |
324 int mallopt(int cmd, int value) __THROW ALIAS("tc_mallopt"); | |
325 #ifdef HAVE_STRUCT_MALLINFO | |
326 struct mallinfo mallinfo(void) __THROW ALIAS("tc_mallinfo"); | |
327 #endif | |
328 size_t malloc_size(void* p) __THROW ALIAS("tc_malloc_size"); | |
329 size_t malloc_usable_size(void* p) __THROW ALIAS("tc_malloc_size"); | |
330 } // extern "C" | |
331 #else // #if defined(__GNUC__) && !defined(__MACH__) | |
332 // Portable wrappers | |
333 void* operator new(size_t size) { return tc_new(size); } | |
334 void operator delete(void* p) __THROW { tc_delete(p); } | |
335 void* operator new[](size_t size) { return tc_newarray(size); } | |
336 void operator delete[](void* p) __THROW { tc_deletearray(p); } | |
337 void* operator new(size_t size, const std::nothrow_t& nt) __THROW { | |
338 return tc_new_nothrow(size, nt); | |
339 } | |
340 void* operator new[](size_t size, const std::nothrow_t& nt) __THROW { | |
341 return tc_newarray_nothrow(size, nt); | |
342 } | |
343 void operator delete(void* ptr, const std::nothrow_t& nt) __THROW { | |
344 return tc_delete_nothrow(ptr, nt); | |
345 } | |
346 void operator delete[](void* ptr, const std::nothrow_t& nt) __THROW { | |
347 return tc_deletearray_nothrow(ptr, nt); | |
348 } | |
349 extern "C" { | |
350 void* malloc(size_t s) __THROW { return tc_malloc(s); } | |
351 void free(void* p) __THROW { tc_free(p); } | |
352 void* realloc(void* p, size_t s) __THROW { return tc_realloc(p, s); } | |
353 void* calloc(size_t n, size_t s) __THROW { return tc_calloc(n, s); } | |
354 void cfree(void* p) __THROW { tc_cfree(p); } | |
355 void* memalign(size_t a, size_t s) __THROW { return tc_memalign(a, s); } | |
356 void* valloc(size_t s) __THROW { return tc_valloc(s); } | |
357 void* pvalloc(size_t s) __THROW { return tc_pvalloc(s); } | |
358 int posix_memalign(void** r, size_t a, size_t s) __THROW { | |
359 return tc_posix_memalign(r, a, s); | |
360 } | |
361 void malloc_stats(void) __THROW { tc_malloc_stats(); } | |
362 int mallopt(int cmd, int v) __THROW { return tc_mallopt(cmd, v); } | |
363 #ifdef HAVE_STRUCT_MALLINFO | |
364 struct mallinfo mallinfo(void) __THROW { return tc_mallinfo(); } | |
365 #endif | |
366 size_t malloc_size(void* p) __THROW { return tc_malloc_size(p); } | |
367 size_t malloc_usable_size(void* p) __THROW { return tc_malloc_size(p); } | |
368 } // extern "C" | |
369 #endif // #if defined(__GNUC__) | |
370 | |
371 // Some library routines on RedHat 9 allocate memory using malloc() | |
372 // and free it using __libc_free() (or vice-versa). Since we provide | |
373 // our own implementations of malloc/free, we need to make sure that | |
374 // the __libc_XXX variants (defined as part of glibc) also point to | |
375 // the same implementations. | |
376 #ifdef __GLIBC__ // only glibc defines __libc_* | |
377 extern "C" { | |
378 #ifdef ALIAS | |
379 void* __libc_malloc(size_t size) ALIAS("tc_malloc"); | |
380 void __libc_free(void* ptr) ALIAS("tc_free"); | |
381 void* __libc_realloc(void* ptr, size_t size) ALIAS("tc_realloc"); | |
382 void* __libc_calloc(size_t n, size_t size) ALIAS("tc_calloc"); | |
383 void __libc_cfree(void* ptr) ALIAS("tc_cfree"); | |
384 void* __libc_memalign(size_t align, size_t s) ALIAS("tc_memalign"); | |
385 void* __libc_valloc(size_t size) ALIAS("tc_valloc"); | |
386 void* __libc_pvalloc(size_t size) ALIAS("tc_pvalloc"); | |
387 int __posix_memalign(void** r, size_t a, size_t s) ALIAS("tc_posix_memalign"); | |
388 #else // #ifdef ALIAS | |
389 void* __libc_malloc(size_t size) { return malloc(size); } | |
390 void __libc_free(void* ptr) { free(ptr); } | |
391 void* __libc_realloc(void* ptr, size_t size) { return realloc(ptr, size); } | |
392 void* __libc_calloc(size_t n, size_t size) { return calloc(n, size); } | |
393 void __libc_cfree(void* ptr) { cfree(ptr); } | |
394 void* __libc_memalign(size_t align, size_t s) { return memalign(align, s); } | |
395 void* __libc_valloc(size_t size) { return valloc(size); } | |
396 void* __libc_pvalloc(size_t size) { return pvalloc(size); } | |
397 int __posix_memalign(void** r, size_t a, size_t s) { | |
398 return posix_memalign(r, a, s); | |
399 } | |
400 #endif // #ifdef ALIAS | |
401 } // extern "C" | |
402 #endif // ifdef __GLIBC__ | |
403 | |
404 #if defined(__GLIBC__) && defined(HAVE_MALLOC_H) | |
405 // If we're using glibc, then override glibc malloc hooks to make sure that even | |
406 // if calls fall through to ptmalloc (due to dlopen() with RTLD_DEEPBIND or what | |
407 // not), ptmalloc will use TCMalloc. | |
408 | |
409 static void* tc_ptmalloc_malloc_hook(size_t size, const void* caller) { | |
410 return tc_malloc(size); | |
411 } | |
412 | |
413 void* (*__MALLOC_HOOK_VOLATILE __malloc_hook)( | |
414 size_t size, const void* caller) = tc_ptmalloc_malloc_hook; | |
415 | |
416 static void* tc_ptmalloc_realloc_hook( | |
417 void* ptr, size_t size, const void* caller) { | |
418 return tc_realloc(ptr, size); | |
419 } | |
420 | |
421 void* (*__MALLOC_HOOK_VOLATILE __realloc_hook)( | |
422 void* ptr, size_t size, const void* caller) = tc_ptmalloc_realloc_hook; | |
423 | |
424 static void tc_ptmalloc_free_hook(void* ptr, const void* caller) { | |
425 tc_free(ptr); | |
426 } | |
427 | |
428 void (*__MALLOC_HOOK_VOLATILE __free_hook)(void* ptr, const void* caller) = tc_p
tmalloc_free_hook; | |
429 | |
430 #endif | |
431 | |
432 #endif // #ifndef _WIN32 | |
433 #undef ALIAS | |
434 | |
435 #endif // #ifndef(WIN32_DO_PATCHING) | |
436 | |
437 | 285 |
438 // ----------------------- IMPLEMENTATION ------------------------------- | 286 // ----------------------- IMPLEMENTATION ------------------------------- |
439 | 287 |
440 static int tc_new_mode = 0; // See tc_set_new_mode(). | 288 static int tc_new_mode = 0; // See tc_set_new_mode(). |
441 | 289 |
442 // Routines such as free() and realloc() catch some erroneous pointers | 290 // Routines such as free() and realloc() catch some erroneous pointers |
443 // passed to them, and invoke the below when they do. (An erroneous pointer | 291 // passed to them, and invoke the below when they do. (An erroneous pointer |
444 // won't be caught if it's within a valid span or a stale span for which | 292 // won't be caught if it's within a valid span or a stale span for which |
445 // the pagemap cache has a non-zero sizeclass.) This is a cheap (source-editing | 293 // the pagemap cache has a non-zero sizeclass.) This is a cheap (source-editing |
446 // required) kind of exception handling for these routines. | 294 // required) kind of exception handling for these routines. |
447 namespace { | 295 namespace { |
448 void InvalidFree(void* ptr) { | 296 void InvalidFree(void* ptr) { |
449 CRASH("Attempt to free invalid pointer: %p\n", ptr); | 297 Log(kCrash, __FILE__, __LINE__, "Attempt to free invalid pointer", ptr); |
450 } | 298 } |
451 | 299 |
452 size_t InvalidGetSizeForRealloc(void* old_ptr) { | 300 size_t InvalidGetSizeForRealloc(const void* old_ptr) { |
453 CRASH("Attempt to realloc invalid pointer: %p\n", old_ptr); | 301 Log(kCrash, __FILE__, __LINE__, |
| 302 "Attempt to realloc invalid pointer", old_ptr); |
454 return 0; | 303 return 0; |
455 } | 304 } |
456 | 305 |
457 size_t InvalidGetAllocatedSize(void* ptr) { | 306 size_t InvalidGetAllocatedSize(const void* ptr) { |
458 CRASH("Attempt to get the size of an invalid pointer: %p\n", ptr); | 307 Log(kCrash, __FILE__, __LINE__, |
| 308 "Attempt to get the size of an invalid pointer", ptr); |
459 return 0; | 309 return 0; |
460 } | 310 } |
461 } // unnamed namespace | 311 } // unnamed namespace |
462 | 312 |
463 // Extract interesting stats | 313 // Extract interesting stats |
464 struct TCMallocStats { | 314 struct TCMallocStats { |
465 uint64_t thread_bytes; // Bytes in thread caches | 315 uint64_t thread_bytes; // Bytes in thread caches |
466 uint64_t central_bytes; // Bytes in central cache | 316 uint64_t central_bytes; // Bytes in central cache |
467 uint64_t transfer_bytes; // Bytes in central transfer cache | 317 uint64_t transfer_bytes; // Bytes in central transfer cache |
468 uint64_t metadata_bytes; // Bytes alloced for metadata | 318 uint64_t metadata_bytes; // Bytes alloced for metadata |
469 PageHeap::Stats pageheap; // Stats from page heap | 319 PageHeap::Stats pageheap; // Stats from page heap |
470 }; | 320 }; |
471 | 321 |
472 // Get stats into "r". Also get per-size-class counts if class_count != NULL | 322 // Get stats into "r". Also get per-size-class counts if class_count != NULL |
473 static void ExtractStats(TCMallocStats* r, uint64_t* class_count) { | 323 static void ExtractStats(TCMallocStats* r, uint64_t* class_count, |
| 324 PageHeap::SmallSpanStats* small_spans, |
| 325 PageHeap::LargeSpanStats* large_spans) { |
474 r->central_bytes = 0; | 326 r->central_bytes = 0; |
475 r->transfer_bytes = 0; | 327 r->transfer_bytes = 0; |
476 for (int cl = 0; cl < kNumClasses; ++cl) { | 328 for (int cl = 0; cl < kNumClasses; ++cl) { |
477 const int length = Static::central_cache()[cl].length(); | 329 const int length = Static::central_cache()[cl].length(); |
478 const int tc_length = Static::central_cache()[cl].tc_length(); | 330 const int tc_length = Static::central_cache()[cl].tc_length(); |
| 331 const size_t cache_overhead = Static::central_cache()[cl].OverheadBytes(); |
479 const size_t size = static_cast<uint64_t>( | 332 const size_t size = static_cast<uint64_t>( |
480 Static::sizemap()->ByteSizeForClass(cl)); | 333 Static::sizemap()->ByteSizeForClass(cl)); |
481 r->central_bytes += (size * length); | 334 r->central_bytes += (size * length) + cache_overhead; |
482 r->transfer_bytes += (size * tc_length); | 335 r->transfer_bytes += (size * tc_length); |
483 if (class_count) class_count[cl] = length + tc_length; | 336 if (class_count) class_count[cl] = length + tc_length; |
484 } | 337 } |
485 | 338 |
486 // Add stats from per-thread heaps | 339 // Add stats from per-thread heaps |
487 r->thread_bytes = 0; | 340 r->thread_bytes = 0; |
488 { // scope | 341 { // scope |
489 SpinLockHolder h(Static::pageheap_lock()); | 342 SpinLockHolder h(Static::pageheap_lock()); |
490 ThreadCache::GetThreadStats(&r->thread_bytes, class_count); | 343 ThreadCache::GetThreadStats(&r->thread_bytes, class_count); |
491 r->metadata_bytes = tcmalloc::metadata_system_bytes(); | 344 r->metadata_bytes = tcmalloc::metadata_system_bytes(); |
492 r->pageheap = Static::pageheap()->stats(); | 345 r->pageheap = Static::pageheap()->stats(); |
| 346 if (small_spans != NULL) { |
| 347 Static::pageheap()->GetSmallSpanStats(small_spans); |
| 348 } |
| 349 if (large_spans != NULL) { |
| 350 Static::pageheap()->GetLargeSpanStats(large_spans); |
| 351 } |
493 } | 352 } |
494 } | 353 } |
495 | 354 |
| 355 static double PagesToMiB(uint64_t pages) { |
| 356 return (pages << kPageShift) / 1048576.0; |
| 357 } |
| 358 |
496 // WRITE stats to "out" | 359 // WRITE stats to "out" |
497 static void DumpStats(TCMalloc_Printer* out, int level) { | 360 static void DumpStats(TCMalloc_Printer* out, int level) { |
498 TCMallocStats stats; | 361 TCMallocStats stats; |
499 uint64_t class_count[kNumClasses]; | 362 uint64_t class_count[kNumClasses]; |
500 ExtractStats(&stats, (level >= 2 ? class_count : NULL)); | 363 PageHeap::SmallSpanStats small; |
| 364 PageHeap::LargeSpanStats large; |
| 365 if (level >= 2) { |
| 366 ExtractStats(&stats, class_count, &small, &large); |
| 367 } else { |
| 368 ExtractStats(&stats, NULL, NULL, NULL); |
| 369 } |
501 | 370 |
502 static const double MiB = 1048576.0; | 371 static const double MiB = 1048576.0; |
503 | 372 |
504 const uint64_t virtual_memory_used = (stats.pageheap.system_bytes | 373 const uint64_t virtual_memory_used = (stats.pageheap.system_bytes |
505 + stats.metadata_bytes); | 374 + stats.metadata_bytes); |
506 const uint64_t physical_memory_used = (virtual_memory_used | 375 const uint64_t physical_memory_used = (virtual_memory_used |
507 - stats.pageheap.unmapped_bytes); | 376 - stats.pageheap.unmapped_bytes); |
508 const uint64_t bytes_in_use_by_app = (physical_memory_used | 377 const uint64_t bytes_in_use_by_app = (physical_memory_used |
509 - stats.metadata_bytes | 378 - stats.metadata_bytes |
510 - stats.pageheap.free_bytes | 379 - stats.pageheap.free_bytes |
(...skipping 63 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
574 cumulative += class_bytes; | 443 cumulative += class_bytes; |
575 out->printf("class %3d [ %8" PRIuS " bytes ] : " | 444 out->printf("class %3d [ %8" PRIuS " bytes ] : " |
576 "%8" PRIu64 " objs; %5.1f MiB; %5.1f cum MiB\n", | 445 "%8" PRIu64 " objs; %5.1f MiB; %5.1f cum MiB\n", |
577 cl, Static::sizemap()->ByteSizeForClass(cl), | 446 cl, Static::sizemap()->ByteSizeForClass(cl), |
578 class_count[cl], | 447 class_count[cl], |
579 class_bytes / MiB, | 448 class_bytes / MiB, |
580 cumulative / MiB); | 449 cumulative / MiB); |
581 } | 450 } |
582 } | 451 } |
583 | 452 |
584 SpinLockHolder h(Static::pageheap_lock()); | 453 // append page heap info |
585 Static::pageheap()->Dump(out); | 454 int nonempty_sizes = 0; |
| 455 for (int s = 0; s < kMaxPages; s++) { |
| 456 if (small.normal_length[s] + small.returned_length[s] > 0) { |
| 457 nonempty_sizes++; |
| 458 } |
| 459 } |
| 460 out->printf("------------------------------------------------\n"); |
| 461 out->printf("PageHeap: %d sizes; %6.1f MiB free; %6.1f MiB unmapped\n", |
| 462 nonempty_sizes, stats.pageheap.free_bytes / MiB, |
| 463 stats.pageheap.unmapped_bytes / MiB); |
| 464 out->printf("------------------------------------------------\n"); |
| 465 uint64_t total_normal = 0; |
| 466 uint64_t total_returned = 0; |
| 467 for (int s = 0; s < kMaxPages; s++) { |
| 468 const int n_length = small.normal_length[s]; |
| 469 const int r_length = small.returned_length[s]; |
| 470 if (n_length + r_length > 0) { |
| 471 uint64_t n_pages = s * n_length; |
| 472 uint64_t r_pages = s * r_length; |
| 473 total_normal += n_pages; |
| 474 total_returned += r_pages; |
| 475 out->printf("%6u pages * %6u spans ~ %6.1f MiB; %6.1f MiB cum" |
| 476 "; unmapped: %6.1f MiB; %6.1f MiB cum\n", |
| 477 s, |
| 478 (n_length + r_length), |
| 479 PagesToMiB(n_pages + r_pages), |
| 480 PagesToMiB(total_normal + total_returned), |
| 481 PagesToMiB(r_pages), |
| 482 PagesToMiB(total_returned)); |
| 483 } |
| 484 } |
| 485 |
| 486 total_normal += large.normal_pages; |
| 487 total_returned += large.returned_pages; |
| 488 out->printf(">255 large * %6u spans ~ %6.1f MiB; %6.1f MiB cum" |
| 489 "; unmapped: %6.1f MiB; %6.1f MiB cum\n", |
| 490 static_cast<unsigned int>(large.spans), |
| 491 PagesToMiB(large.normal_pages + large.returned_pages), |
| 492 PagesToMiB(total_normal + total_returned), |
| 493 PagesToMiB(large.returned_pages), |
| 494 PagesToMiB(total_returned)); |
586 } | 495 } |
587 } | 496 } |
588 | 497 |
589 static void PrintStats(int level) { | 498 static void PrintStats(int level) { |
590 const int kBufferSize = 16 << 10; | 499 const int kBufferSize = 16 << 10; |
591 char* buffer = new char[kBufferSize]; | 500 char* buffer = new char[kBufferSize]; |
592 TCMalloc_Printer printer(buffer, kBufferSize); | 501 TCMalloc_Printer printer(buffer, kBufferSize); |
593 DumpStats(&printer, level); | 502 DumpStats(&printer, level); |
594 write(STDERR_FILENO, buffer, strlen(buffer)); | 503 write(STDERR_FILENO, buffer, strlen(buffer)); |
595 delete[] buffer; | 504 delete[] buffer; |
596 } | 505 } |
597 | 506 |
598 static void** DumpHeapGrowthStackTraces() { | 507 static void** DumpHeapGrowthStackTraces() { |
599 // Count how much space we need | 508 // Count how much space we need |
600 int needed_slots = 0; | 509 int needed_slots = 0; |
601 { | 510 { |
602 SpinLockHolder h(Static::pageheap_lock()); | 511 SpinLockHolder h(Static::pageheap_lock()); |
603 for (StackTrace* t = Static::growth_stacks(); | 512 for (StackTrace* t = Static::growth_stacks(); |
604 t != NULL; | 513 t != NULL; |
605 t = reinterpret_cast<StackTrace*>( | 514 t = reinterpret_cast<StackTrace*>( |
606 t->stack[tcmalloc::kMaxStackDepth-1])) { | 515 t->stack[tcmalloc::kMaxStackDepth-1])) { |
607 needed_slots += 3 + t->depth; | 516 needed_slots += 3 + t->depth; |
608 } | 517 } |
609 needed_slots += 100; // Slop in case list grows | 518 needed_slots += 100; // Slop in case list grows |
610 needed_slots += needed_slots/8; // An extra 12.5% slop | 519 needed_slots += needed_slots/8; // An extra 12.5% slop |
611 } | 520 } |
612 | 521 |
613 void** result = new void*[needed_slots]; | 522 void** result = new void*[needed_slots]; |
614 if (result == NULL) { | 523 if (result == NULL) { |
615 MESSAGE("tcmalloc: allocation failed for stack trace slots", | 524 Log(kLog, __FILE__, __LINE__, |
616 needed_slots * sizeof(*result)); | 525 "tcmalloc: allocation failed for stack trace slots", |
| 526 needed_slots * sizeof(*result)); |
617 return NULL; | 527 return NULL; |
618 } | 528 } |
619 | 529 |
620 SpinLockHolder h(Static::pageheap_lock()); | 530 SpinLockHolder h(Static::pageheap_lock()); |
621 int used_slots = 0; | 531 int used_slots = 0; |
622 for (StackTrace* t = Static::growth_stacks(); | 532 for (StackTrace* t = Static::growth_stacks(); |
623 t != NULL; | 533 t != NULL; |
624 t = reinterpret_cast<StackTrace*>( | 534 t = reinterpret_cast<StackTrace*>( |
625 t->stack[tcmalloc::kMaxStackDepth-1])) { | 535 t->stack[tcmalloc::kMaxStackDepth-1])) { |
626 ASSERT(used_slots < needed_slots); // Need to leave room for terminator | 536 ASSERT(used_slots < needed_slots); // Need to leave room for terminator |
(...skipping 105 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
732 | 642 |
733 virtual void Ranges(void* arg, RangeFunction func) { | 643 virtual void Ranges(void* arg, RangeFunction func) { |
734 IterateOverRanges(arg, func); | 644 IterateOverRanges(arg, func); |
735 } | 645 } |
736 | 646 |
737 virtual bool GetNumericProperty(const char* name, size_t* value) { | 647 virtual bool GetNumericProperty(const char* name, size_t* value) { |
738 ASSERT(name != NULL); | 648 ASSERT(name != NULL); |
739 | 649 |
740 if (strcmp(name, "generic.current_allocated_bytes") == 0) { | 650 if (strcmp(name, "generic.current_allocated_bytes") == 0) { |
741 TCMallocStats stats; | 651 TCMallocStats stats; |
742 ExtractStats(&stats, NULL); | 652 ExtractStats(&stats, NULL, NULL, NULL); |
743 *value = stats.pageheap.system_bytes | 653 *value = stats.pageheap.system_bytes |
744 - stats.thread_bytes | 654 - stats.thread_bytes |
745 - stats.central_bytes | 655 - stats.central_bytes |
746 - stats.transfer_bytes | 656 - stats.transfer_bytes |
747 - stats.pageheap.free_bytes | 657 - stats.pageheap.free_bytes |
748 - stats.pageheap.unmapped_bytes; | 658 - stats.pageheap.unmapped_bytes; |
749 return true; | 659 return true; |
750 } | 660 } |
751 | 661 |
752 if (strcmp(name, "generic.heap_size") == 0) { | 662 if (strcmp(name, "generic.heap_size") == 0) { |
753 TCMallocStats stats; | 663 TCMallocStats stats; |
754 ExtractStats(&stats, NULL); | 664 ExtractStats(&stats, NULL, NULL, NULL); |
755 *value = stats.pageheap.system_bytes; | 665 *value = stats.pageheap.system_bytes; |
756 return true; | 666 return true; |
757 } | 667 } |
758 | 668 |
759 if (strcmp(name, "tcmalloc.slack_bytes") == 0) { | 669 if (strcmp(name, "tcmalloc.slack_bytes") == 0) { |
760 // Kept for backwards compatibility. Now defined externally as: | 670 // Kept for backwards compatibility. Now defined externally as: |
761 // pageheap_free_bytes + pageheap_unmapped_bytes. | 671 // pageheap_free_bytes + pageheap_unmapped_bytes. |
762 SpinLockHolder l(Static::pageheap_lock()); | 672 SpinLockHolder l(Static::pageheap_lock()); |
763 PageHeap::Stats stats = Static::pageheap()->stats(); | 673 PageHeap::Stats stats = Static::pageheap()->stats(); |
764 *value = stats.free_bytes + stats.unmapped_bytes; | 674 *value = stats.free_bytes + stats.unmapped_bytes; |
(...skipping 13 matching lines...) Expand all Loading... |
778 } | 688 } |
779 | 689 |
780 if (strcmp(name, "tcmalloc.max_total_thread_cache_bytes") == 0) { | 690 if (strcmp(name, "tcmalloc.max_total_thread_cache_bytes") == 0) { |
781 SpinLockHolder l(Static::pageheap_lock()); | 691 SpinLockHolder l(Static::pageheap_lock()); |
782 *value = ThreadCache::overall_thread_cache_size(); | 692 *value = ThreadCache::overall_thread_cache_size(); |
783 return true; | 693 return true; |
784 } | 694 } |
785 | 695 |
786 if (strcmp(name, "tcmalloc.current_total_thread_cache_bytes") == 0) { | 696 if (strcmp(name, "tcmalloc.current_total_thread_cache_bytes") == 0) { |
787 TCMallocStats stats; | 697 TCMallocStats stats; |
788 ExtractStats(&stats, NULL); | 698 ExtractStats(&stats, NULL, NULL, NULL); |
789 *value = stats.thread_bytes; | 699 *value = stats.thread_bytes; |
790 return true; | 700 return true; |
791 } | 701 } |
792 | 702 |
793 return false; | 703 return false; |
794 } | 704 } |
795 | 705 |
796 virtual bool SetNumericProperty(const char* name, size_t value) { | 706 virtual bool SetNumericProperty(const char* name, size_t value) { |
797 ASSERT(name != NULL); | 707 ASSERT(name != NULL); |
798 | 708 |
(...skipping 60 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
859 const size_t alloc_size = Static::sizemap()->ByteSizeForClass(cl); | 769 const size_t alloc_size = Static::sizemap()->ByteSizeForClass(cl); |
860 return alloc_size; | 770 return alloc_size; |
861 } else { | 771 } else { |
862 return tcmalloc::pages(size) << kPageShift; | 772 return tcmalloc::pages(size) << kPageShift; |
863 } | 773 } |
864 } | 774 } |
865 | 775 |
866 // This just calls GetSizeWithCallback, but because that's in an | 776 // This just calls GetSizeWithCallback, but because that's in an |
867 // unnamed namespace, we need to move the definition below it in the | 777 // unnamed namespace, we need to move the definition below it in the |
868 // file. | 778 // file. |
869 virtual size_t GetAllocatedSize(void* ptr); | 779 virtual size_t GetAllocatedSize(const void* ptr); |
| 780 |
| 781 // This duplicates some of the logic in GetSizeWithCallback, but is |
| 782 // faster. This is important on OS X, where this function is called |
| 783 // on every allocation operation. |
| 784 virtual Ownership GetOwnership(const void* ptr) { |
| 785 const PageID p = reinterpret_cast<uintptr_t>(ptr) >> kPageShift; |
| 786 // The rest of tcmalloc assumes that all allocated pointers use at |
| 787 // most kAddressBits bits. If ptr doesn't, then it definitely |
| 788 // wasn't alloacted by tcmalloc. |
| 789 if ((p >> (kAddressBits - kPageShift)) > 0) { |
| 790 return kNotOwned; |
| 791 } |
| 792 size_t cl = Static::pageheap()->GetSizeClassIfCached(p); |
| 793 if (cl != 0) { |
| 794 return kOwned; |
| 795 } |
| 796 const Span *span = Static::pageheap()->GetDescriptor(p); |
| 797 return span ? kOwned : kNotOwned; |
| 798 } |
870 | 799 |
871 virtual void GetFreeListSizes(vector<MallocExtension::FreeListInfo>* v) { | 800 virtual void GetFreeListSizes(vector<MallocExtension::FreeListInfo>* v) { |
872 static const char* kCentralCacheType = "tcmalloc.central"; | 801 static const char* kCentralCacheType = "tcmalloc.central"; |
873 static const char* kTransferCacheType = "tcmalloc.transfer"; | 802 static const char* kTransferCacheType = "tcmalloc.transfer"; |
874 static const char* kThreadCacheType = "tcmalloc.thread"; | 803 static const char* kThreadCacheType = "tcmalloc.thread"; |
875 static const char* kPageHeapType = "tcmalloc.page"; | 804 static const char* kPageHeapType = "tcmalloc.page"; |
876 static const char* kPageHeapUnmappedType = "tcmalloc.page_unmapped"; | 805 static const char* kPageHeapUnmappedType = "tcmalloc.page_unmapped"; |
877 static const char* kLargeSpanType = "tcmalloc.large"; | 806 static const char* kLargeSpanType = "tcmalloc.large"; |
878 static const char* kLargeUnmappedSpanType = "tcmalloc.large_unmapped"; | 807 static const char* kLargeUnmappedSpanType = "tcmalloc.large_unmapped"; |
879 | 808 |
(...skipping 34 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
914 MallocExtension::FreeListInfo i; | 843 MallocExtension::FreeListInfo i; |
915 i.min_object_size = prev_class_size + 1; | 844 i.min_object_size = prev_class_size + 1; |
916 i.max_object_size = Static::sizemap()->ByteSizeForClass(cl); | 845 i.max_object_size = Static::sizemap()->ByteSizeForClass(cl); |
917 i.total_bytes_free = | 846 i.total_bytes_free = |
918 class_count[cl] * Static::sizemap()->ByteSizeForClass(cl); | 847 class_count[cl] * Static::sizemap()->ByteSizeForClass(cl); |
919 i.type = kThreadCacheType; | 848 i.type = kThreadCacheType; |
920 v->push_back(i); | 849 v->push_back(i); |
921 } | 850 } |
922 | 851 |
923 // append page heap info | 852 // append page heap info |
924 int64 page_count_normal[kMaxPages]; | 853 PageHeap::SmallSpanStats small; |
925 int64 page_count_returned[kMaxPages]; | 854 PageHeap::LargeSpanStats large; |
926 int64 span_count_normal; | |
927 int64 span_count_returned; | |
928 { | 855 { |
929 SpinLockHolder h(Static::pageheap_lock()); | 856 SpinLockHolder h(Static::pageheap_lock()); |
930 Static::pageheap()->GetClassSizes(page_count_normal, | 857 Static::pageheap()->GetSmallSpanStats(&small); |
931 page_count_returned, | 858 Static::pageheap()->GetLargeSpanStats(&large); |
932 &span_count_normal, | |
933 &span_count_returned); | |
934 } | 859 } |
935 | 860 |
936 // spans: mapped | 861 // large spans: mapped |
937 MallocExtension::FreeListInfo span_info; | 862 MallocExtension::FreeListInfo span_info; |
938 span_info.type = kLargeSpanType; | 863 span_info.type = kLargeSpanType; |
939 span_info.max_object_size = (numeric_limits<size_t>::max)(); | 864 span_info.max_object_size = (numeric_limits<size_t>::max)(); |
940 span_info.min_object_size = kMaxPages << kPageShift; | 865 span_info.min_object_size = kMaxPages << kPageShift; |
941 span_info.total_bytes_free = span_count_normal << kPageShift; | 866 span_info.total_bytes_free = large.normal_pages << kPageShift; |
942 v->push_back(span_info); | 867 v->push_back(span_info); |
943 | 868 |
944 // spans: unmapped | 869 // large spans: unmapped |
945 span_info.type = kLargeUnmappedSpanType; | 870 span_info.type = kLargeUnmappedSpanType; |
946 span_info.total_bytes_free = span_count_returned << kPageShift; | 871 span_info.total_bytes_free = large.returned_pages << kPageShift; |
947 v->push_back(span_info); | 872 v->push_back(span_info); |
948 | 873 |
| 874 // small spans |
949 for (int s = 1; s < kMaxPages; s++) { | 875 for (int s = 1; s < kMaxPages; s++) { |
950 MallocExtension::FreeListInfo i; | 876 MallocExtension::FreeListInfo i; |
951 i.max_object_size = (s << kPageShift); | 877 i.max_object_size = (s << kPageShift); |
952 i.min_object_size = ((s - 1) << kPageShift); | 878 i.min_object_size = ((s - 1) << kPageShift); |
953 | 879 |
954 i.type = kPageHeapType; | 880 i.type = kPageHeapType; |
955 i.total_bytes_free = (s << kPageShift) * page_count_normal[s]; | 881 i.total_bytes_free = (s << kPageShift) * small.normal_length[s]; |
956 v->push_back(i); | 882 v->push_back(i); |
957 | 883 |
958 i.type = kPageHeapUnmappedType; | 884 i.type = kPageHeapUnmappedType; |
959 i.total_bytes_free = (s << kPageShift) * page_count_returned[s]; | 885 i.total_bytes_free = (s << kPageShift) * small.returned_length[s]; |
960 v->push_back(i); | 886 v->push_back(i); |
961 } | 887 } |
962 } | 888 } |
963 }; | 889 }; |
964 | 890 |
965 // The constructor allocates an object to ensure that initialization | 891 // The constructor allocates an object to ensure that initialization |
966 // runs before main(), and therefore we do not have a chance to become | 892 // runs before main(), and therefore we do not have a chance to become |
967 // multi-threaded before initialization. We also create the TSD key | 893 // multi-threaded before initialization. We also create the TSD key |
968 // here. Presumably by the time this constructor runs, glibc is in | 894 // here. Presumably by the time this constructor runs, glibc is in |
969 // good enough shape to handle pthread_key_create(). | 895 // good enough shape to handle pthread_key_create(). |
970 // | 896 // |
971 // The constructor also takes the opportunity to tell STL to use | 897 // The constructor also takes the opportunity to tell STL to use |
972 // tcmalloc. We want to do this early, before construct time, so | 898 // tcmalloc. We want to do this early, before construct time, so |
973 // all user STL allocations go through tcmalloc (which works really | 899 // all user STL allocations go through tcmalloc (which works really |
974 // well for STL). | 900 // well for STL). |
975 // | 901 // |
976 // The destructor prints stats when the program exits. | 902 // The destructor prints stats when the program exits. |
977 static int tcmallocguard_refcount = 0; // no lock needed: runs before main() | 903 static int tcmallocguard_refcount = 0; // no lock needed: runs before main() |
978 TCMallocGuard::TCMallocGuard() { | 904 TCMallocGuard::TCMallocGuard() { |
979 if (tcmallocguard_refcount++ == 0) { | 905 if (tcmallocguard_refcount++ == 0) { |
980 #ifdef HAVE_TLS // this is true if the cc/ld/libc combo support TLS | 906 #ifdef HAVE_TLS // this is true if the cc/ld/libc combo support TLS |
981 // Check whether the kernel also supports TLS (needs to happen at runtime) | 907 // Check whether the kernel also supports TLS (needs to happen at runtime) |
982 tcmalloc::CheckIfKernelSupportsTLS(); | 908 tcmalloc::CheckIfKernelSupportsTLS(); |
983 #endif | 909 #endif |
984 #ifdef WIN32_DO_PATCHING | 910 ReplaceSystemAlloc(); // defined in libc_override_*.h |
985 // patch the windows VirtualAlloc, etc. | |
986 PatchWindowsFunctions(); // defined in windows/patch_functions.cc | |
987 #endif | |
988 tc_free(tc_malloc(1)); | 911 tc_free(tc_malloc(1)); |
989 ThreadCache::InitTSD(); | 912 ThreadCache::InitTSD(); |
990 tc_free(tc_malloc(1)); | 913 tc_free(tc_malloc(1)); |
991 // Either we, or debugallocation.cc, or valgrind will control memory | 914 // Either we, or debugallocation.cc, or valgrind will control memory |
992 // management. We register our extension if we're the winner. | 915 // management. We register our extension if we're the winner. |
993 #ifdef TCMALLOC_USING_DEBUGALLOCATION | 916 #ifdef TCMALLOC_USING_DEBUGALLOCATION |
994 // Let debugallocation register its extension. | 917 // Let debugallocation register its extension. |
995 #else | 918 #else |
996 if (RunningOnValgrind()) { | 919 if (RunningOnValgrind()) { |
997 // Let Valgrind uses its own malloc (so don't register our extension). | 920 // Let Valgrind uses its own malloc (so don't register our extension). |
(...skipping 76 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1074 (kPageSize > FLAGS_tcmalloc_large_alloc_report_threshold | 997 (kPageSize > FLAGS_tcmalloc_large_alloc_report_threshold |
1075 ? kPageSize : FLAGS_tcmalloc_large_alloc_report_threshold); | 998 ? kPageSize : FLAGS_tcmalloc_large_alloc_report_threshold); |
1076 | 999 |
1077 static void ReportLargeAlloc(Length num_pages, void* result) { | 1000 static void ReportLargeAlloc(Length num_pages, void* result) { |
1078 StackTrace stack; | 1001 StackTrace stack; |
1079 stack.depth = GetStackTrace(stack.stack, tcmalloc::kMaxStackDepth, 1); | 1002 stack.depth = GetStackTrace(stack.stack, tcmalloc::kMaxStackDepth, 1); |
1080 | 1003 |
1081 static const int N = 1000; | 1004 static const int N = 1000; |
1082 char buffer[N]; | 1005 char buffer[N]; |
1083 TCMalloc_Printer printer(buffer, N); | 1006 TCMalloc_Printer printer(buffer, N); |
1084 printer.printf("tcmalloc: large alloc %llu bytes == %p @ ", | 1007 printer.printf("tcmalloc: large alloc %"PRIu64" bytes == %p @ ", |
1085 static_cast<unsigned long long>(num_pages) << kPageShift, | 1008 static_cast<uint64>(num_pages) << kPageShift, |
1086 result); | 1009 result); |
1087 for (int i = 0; i < stack.depth; i++) { | 1010 for (int i = 0; i < stack.depth; i++) { |
1088 printer.printf(" %p", stack.stack[i]); | 1011 printer.printf(" %p", stack.stack[i]); |
1089 } | 1012 } |
1090 printer.printf("\n"); | 1013 printer.printf("\n"); |
1091 write(STDERR_FILENO, buffer, strlen(buffer)); | 1014 write(STDERR_FILENO, buffer, strlen(buffer)); |
1092 } | 1015 } |
1093 | 1016 |
1094 inline void* cpp_alloc(size_t size, bool nothrow); | 1017 inline void* cpp_alloc(size_t size, bool nothrow); |
1095 inline void* do_malloc(size_t size); | 1018 inline void* do_malloc(size_t size); |
(...skipping 68 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1164 // TODO(jar): If this has any detectable performance impact, it can be | 1087 // TODO(jar): If this has any detectable performance impact, it can be |
1165 // optimized by only tallying sizes if the profiler was activated to recall | 1088 // optimized by only tallying sizes if the profiler was activated to recall |
1166 // these tallies. I don't think this is performance critical, but we really | 1089 // these tallies. I don't think this is performance critical, but we really |
1167 // should measure it. | 1090 // should measure it. |
1168 heap->AddToByteAllocatedTotal(size); // Chromium profiling. | 1091 heap->AddToByteAllocatedTotal(size); // Chromium profiling. |
1169 | 1092 |
1170 if ((FLAGS_tcmalloc_sample_parameter > 0) && heap->SampleAllocation(size)) { | 1093 if ((FLAGS_tcmalloc_sample_parameter > 0) && heap->SampleAllocation(size)) { |
1171 ret = DoSampledAllocation(size); | 1094 ret = DoSampledAllocation(size); |
1172 MarkAllocatedRegion(ret); | 1095 MarkAllocatedRegion(ret); |
1173 } else { | 1096 } else { |
1174 // The common case, and also the simplest. This just pops the | 1097 // The common case, and also the simplest. This just pops the |
1175 // size-appropriate freelist, after replenishing it if it's empty. | 1098 // size-appropriate freelist, after replenishing it if it's empty. |
1176 ret = CheckedMallocResult(heap->Allocate(size, cl)); | 1099 ret = CheckedMallocResult(heap->Allocate(size, cl)); |
1177 } | 1100 } |
1178 } else { | 1101 } else { |
1179 ret = do_malloc_pages(heap, size); | 1102 ret = do_malloc_pages(heap, size); |
1180 MarkAllocatedRegion(ret); | 1103 MarkAllocatedRegion(ret); |
1181 } | 1104 } |
1182 if (ret == NULL) errno = ENOMEM; | 1105 if (ret == NULL) errno = ENOMEM; |
1183 return ret; | 1106 return ret; |
1184 } | 1107 } |
(...skipping 12 matching lines...) Expand all Loading... |
1197 | 1120 |
1198 static inline ThreadCache* GetCacheIfPresent() { | 1121 static inline ThreadCache* GetCacheIfPresent() { |
1199 void* const p = ThreadCache::GetCacheIfPresent(); | 1122 void* const p = ThreadCache::GetCacheIfPresent(); |
1200 return reinterpret_cast<ThreadCache*>(p); | 1123 return reinterpret_cast<ThreadCache*>(p); |
1201 } | 1124 } |
1202 | 1125 |
1203 // This lets you call back to a given function pointer if ptr is invalid. | 1126 // This lets you call back to a given function pointer if ptr is invalid. |
1204 // It is used primarily by windows code which wants a specialized callback. | 1127 // It is used primarily by windows code which wants a specialized callback. |
1205 inline void do_free_with_callback(void* ptr, void (*invalid_free_fn)(void*)) { | 1128 inline void do_free_with_callback(void* ptr, void (*invalid_free_fn)(void*)) { |
1206 if (ptr == NULL) return; | 1129 if (ptr == NULL) return; |
1207 ASSERT(Static::pageheap() != NULL); // Should not call free() before malloc() | 1130 if (Static::pageheap() == NULL) { |
| 1131 // We called free() before malloc(). This can occur if the |
| 1132 // (system) malloc() is called before tcmalloc is loaded, and then |
| 1133 // free() is called after tcmalloc is loaded (and tc_free has |
| 1134 // replaced free), but before the global constructor has run that |
| 1135 // sets up the tcmalloc data structures. |
| 1136 (*invalid_free_fn)(ptr); // Decide how to handle the bad free request |
| 1137 return; |
| 1138 } |
1208 const PageID p = reinterpret_cast<uintptr_t>(ptr) >> kPageShift; | 1139 const PageID p = reinterpret_cast<uintptr_t>(ptr) >> kPageShift; |
1209 Span* span = NULL; | 1140 Span* span = NULL; |
1210 size_t cl = Static::pageheap()->GetSizeClassIfCached(p); | 1141 size_t cl = Static::pageheap()->GetSizeClassIfCached(p); |
1211 | 1142 |
1212 if (cl == 0) { | 1143 if (cl == 0) { |
1213 span = Static::pageheap()->GetDescriptor(p); | 1144 span = Static::pageheap()->GetDescriptor(p); |
1214 if (!span) { | 1145 if (!span) { |
1215 // span can be NULL because the pointer passed in is invalid | 1146 // span can be NULL because the pointer passed in is invalid |
1216 // (not something returned by malloc or friends), or because the | 1147 // (not something returned by malloc or friends), or because the |
1217 // pointer was allocated with some other allocator besides | 1148 // pointer was allocated with some other allocator besides |
(...skipping 32 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1250 } | 1181 } |
1251 Static::pageheap()->Delete(span); | 1182 Static::pageheap()->Delete(span); |
1252 } | 1183 } |
1253 } | 1184 } |
1254 | 1185 |
1255 // The default "do_free" that uses the default callback. | 1186 // The default "do_free" that uses the default callback. |
1256 inline void do_free(void* ptr) { | 1187 inline void do_free(void* ptr) { |
1257 return do_free_with_callback(ptr, &InvalidFree); | 1188 return do_free_with_callback(ptr, &InvalidFree); |
1258 } | 1189 } |
1259 | 1190 |
1260 inline size_t GetSizeWithCallback(void* ptr, | 1191 // NOTE: some logic here is duplicated in GetOwnership (above), for |
1261 size_t (*invalid_getsize_fn)(void*)) { | 1192 // speed. If you change this function, look at that one too. |
| 1193 inline size_t GetSizeWithCallback(const void* ptr, |
| 1194 size_t (*invalid_getsize_fn)(const void*)) { |
1262 if (ptr == NULL) | 1195 if (ptr == NULL) |
1263 return 0; | 1196 return 0; |
1264 const PageID p = reinterpret_cast<uintptr_t>(ptr) >> kPageShift; | 1197 const PageID p = reinterpret_cast<uintptr_t>(ptr) >> kPageShift; |
1265 size_t cl = Static::pageheap()->GetSizeClassIfCached(p); | 1198 size_t cl = Static::pageheap()->GetSizeClassIfCached(p); |
1266 if (cl != 0) { | 1199 if (cl != 0) { |
1267 return Static::sizemap()->ByteSizeForClass(cl); | 1200 return Static::sizemap()->ByteSizeForClass(cl); |
1268 } else { | 1201 } else { |
1269 Span *span = Static::pageheap()->GetDescriptor(p); | 1202 const Span *span = Static::pageheap()->GetDescriptor(p); |
1270 if (span == NULL) { // means we do not own this memory | 1203 if (span == NULL) { // means we do not own this memory |
1271 return (*invalid_getsize_fn)(ptr); | 1204 return (*invalid_getsize_fn)(ptr); |
1272 } else if (span->sizeclass != 0) { | 1205 } else if (span->sizeclass != 0) { |
1273 Static::pageheap()->CacheSizeClass(p, span->sizeclass); | 1206 Static::pageheap()->CacheSizeClass(p, span->sizeclass); |
1274 return Static::sizemap()->ByteSizeForClass(span->sizeclass); | 1207 return Static::sizemap()->ByteSizeForClass(span->sizeclass); |
1275 } else { | 1208 } else { |
1276 return span->length << kPageShift; | 1209 return span->length << kPageShift; |
1277 } | 1210 } |
1278 } | 1211 } |
1279 } | 1212 } |
1280 | 1213 |
1281 // This lets you call back to a given function pointer if ptr is invalid. | 1214 // This lets you call back to a given function pointer if ptr is invalid. |
1282 // It is used primarily by windows code which wants a specialized callback. | 1215 // It is used primarily by windows code which wants a specialized callback. |
1283 inline void* do_realloc_with_callback( | 1216 inline void* do_realloc_with_callback( |
1284 void* old_ptr, size_t new_size, | 1217 void* old_ptr, size_t new_size, |
1285 void (*invalid_free_fn)(void*), | 1218 void (*invalid_free_fn)(void*), |
1286 size_t (*invalid_get_size_fn)(void*)) { | 1219 size_t (*invalid_get_size_fn)(const void*)) { |
1287 AddRoomForMark(&new_size); | 1220 AddRoomForMark(&new_size); |
1288 // Get the size of the old entry | 1221 // Get the size of the old entry |
1289 const size_t old_size = GetSizeWithCallback(old_ptr, invalid_get_size_fn); | 1222 const size_t old_size = GetSizeWithCallback(old_ptr, invalid_get_size_fn); |
1290 | 1223 |
1291 // Reallocate if the new size is larger than the old size, | 1224 // Reallocate if the new size is larger than the old size, |
1292 // or if the new size is significantly smaller than the old size. | 1225 // or if the new size is significantly smaller than the old size. |
1293 // We do hysteresis to avoid resizing ping-pongs: | 1226 // We do hysteresis to avoid resizing ping-pongs: |
1294 // . If we need to grow, grow to max(new_size, old_size * 1.X) | 1227 // . If we need to grow, grow to max(new_size, old_size * 1.X) |
1295 // . Don't shrink unless new_size < old_size * 0.Y | 1228 // . Don't shrink unless new_size < old_size * 0.Y |
1296 // X and Y trade-off time for wasted space. For now we do 1.25 and 0.5. | 1229 // X and Y trade-off time for wasted space. For now we do 1.25 and 0.5. |
(...skipping 125 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1422 PrintStats(1); | 1355 PrintStats(1); |
1423 } | 1356 } |
1424 | 1357 |
1425 inline int do_mallopt(int cmd, int value) { | 1358 inline int do_mallopt(int cmd, int value) { |
1426 return 1; // Indicates error | 1359 return 1; // Indicates error |
1427 } | 1360 } |
1428 | 1361 |
1429 #ifdef HAVE_STRUCT_MALLINFO | 1362 #ifdef HAVE_STRUCT_MALLINFO |
1430 inline struct mallinfo do_mallinfo() { | 1363 inline struct mallinfo do_mallinfo() { |
1431 TCMallocStats stats; | 1364 TCMallocStats stats; |
1432 ExtractStats(&stats, NULL); | 1365 ExtractStats(&stats, NULL, NULL, NULL); |
1433 | 1366 |
1434 // Just some of the fields are filled in. | 1367 // Just some of the fields are filled in. |
1435 struct mallinfo info; | 1368 struct mallinfo info; |
1436 memset(&info, 0, sizeof(info)); | 1369 memset(&info, 0, sizeof(info)); |
1437 | 1370 |
1438 // Unfortunately, the struct contains "int" field, so some of the | 1371 // Unfortunately, the struct contains "int" field, so some of the |
1439 // size values will be truncated. | 1372 // size values will be truncated. |
1440 info.arena = static_cast<int>(stats.pageheap.system_bytes); | 1373 info.arena = static_cast<int>(stats.pageheap.system_bytes); |
1441 info.fsmblks = static_cast<int>(stats.thread_bytes | 1374 info.fsmblks = static_cast<int>(stats.thread_bytes |
1442 + stats.central_bytes | 1375 + stats.central_bytes |
(...skipping 103 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1546 } else { // allocation success | 1479 } else { // allocation success |
1547 return p; | 1480 return p; |
1548 } | 1481 } |
1549 #endif // PREANSINEW | 1482 #endif // PREANSINEW |
1550 } | 1483 } |
1551 } | 1484 } |
1552 | 1485 |
1553 } // end unnamed namespace | 1486 } // end unnamed namespace |
1554 | 1487 |
1555 // As promised, the definition of this function, declared above. | 1488 // As promised, the definition of this function, declared above. |
1556 size_t TCMallocImplementation::GetAllocatedSize(void* ptr) { | 1489 size_t TCMallocImplementation::GetAllocatedSize(const void* ptr) { |
| 1490 ASSERT(TCMallocImplementation::GetOwnership(ptr) |
| 1491 != TCMallocImplementation::kNotOwned); |
1557 return ExcludeSpaceForMark( | 1492 return ExcludeSpaceForMark( |
1558 GetSizeWithCallback(ptr, &InvalidGetAllocatedSize)); | 1493 GetSizeWithCallback(ptr, &InvalidGetAllocatedSize)); |
1559 } | 1494 } |
1560 | 1495 |
1561 void TCMallocImplementation::MarkThreadBusy() { | 1496 void TCMallocImplementation::MarkThreadBusy() { |
1562 // Allocate to force the creation of a thread cache, but avoid | 1497 // Allocate to force the creation of a thread cache, but avoid |
1563 // invoking any hooks. | 1498 // invoking any hooks. |
1564 do_free(do_malloc(0)); | 1499 do_free(do_malloc(0)); |
1565 } | 1500 } |
1566 | 1501 |
(...skipping 178 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1745 return do_mallopt(cmd, value); | 1680 return do_mallopt(cmd, value); |
1746 } | 1681 } |
1747 | 1682 |
1748 #ifdef HAVE_STRUCT_MALLINFO | 1683 #ifdef HAVE_STRUCT_MALLINFO |
1749 extern "C" PERFTOOLS_DLL_DECL struct mallinfo tc_mallinfo(void) __THROW { | 1684 extern "C" PERFTOOLS_DLL_DECL struct mallinfo tc_mallinfo(void) __THROW { |
1750 return do_mallinfo(); | 1685 return do_mallinfo(); |
1751 } | 1686 } |
1752 #endif | 1687 #endif |
1753 | 1688 |
1754 extern "C" PERFTOOLS_DLL_DECL size_t tc_malloc_size(void* ptr) __THROW { | 1689 extern "C" PERFTOOLS_DLL_DECL size_t tc_malloc_size(void* ptr) __THROW { |
1755 return GetSizeWithCallback(ptr, &InvalidGetAllocatedSize); | 1690 return MallocExtension::instance()->GetAllocatedSize(ptr); |
1756 } | 1691 } |
1757 | 1692 |
1758 | |
1759 // Override __libc_memalign in libc on linux boxes specially. | |
1760 // They have a bug in libc that causes them to (very rarely) allocate | |
1761 // with __libc_memalign() yet deallocate with free() and the | |
1762 // definitions above don't catch it. | |
1763 // This function is an exception to the rule of calling MallocHook method | |
1764 // from the stack frame of the allocation function; | |
1765 // heap-checker handles this special case explicitly. | |
1766 static void *MemalignOverride(size_t align, size_t size, const void *caller) | |
1767 __THROW ATTRIBUTE_SECTION(google_malloc); | |
1768 | |
1769 static void *MemalignOverride(size_t align, size_t size, const void *caller) | |
1770 __THROW { | |
1771 void* result = do_memalign_or_cpp_memalign(align, size); | |
1772 MallocHook::InvokeNewHook(result, size); | |
1773 return result; | |
1774 } | |
1775 void *(*__MALLOC_HOOK_VOLATILE __memalign_hook)(size_t, size_t, const void *) =
MemalignOverride; | |
1776 #endif // TCMALLOC_USING_DEBUGALLOCATION | 1693 #endif // TCMALLOC_USING_DEBUGALLOCATION |
1777 | 1694 |
1778 // ---Double free() debugging implementation ----------------------------------- | 1695 // ---Double free() debugging implementation ----------------------------------- |
1779 // We will put a mark at the extreme end of each allocation block. We make | 1696 // We will put a mark at the extreme end of each allocation block. We make |
1780 // sure that we always allocate enough "extra memory" that we can fit in the | 1697 // sure that we always allocate enough "extra memory" that we can fit in the |
1781 // mark, and still provide the requested usable region. If ever that mark is | 1698 // mark, and still provide the requested usable region. If ever that mark is |
1782 // not as expected, then we know that the user is corrupting memory beyond their | 1699 // not as expected, then we know that the user is corrupting memory beyond their |
1783 // request size, or that they have called free a second time without having | 1700 // request size, or that they have called free a second time without having |
1784 // the memory allocated (again). This allows us to spot most double free()s, | 1701 // the memory allocated (again). This allows us to spot most double free()s, |
1785 // but some can "slip by" or confuse our logic if the caller reallocates memory | 1702 // but some can "slip by" or confuse our logic if the caller reallocates memory |
(...skipping 30 matching lines...) Expand all Loading... |
1816 static void ValidateAllocatedRegion(void* ptr, size_t cl) {} | 1733 static void ValidateAllocatedRegion(void* ptr, size_t cl) {} |
1817 | 1734 |
1818 #else // TCMALLOC_VALIDATION | 1735 #else // TCMALLOC_VALIDATION |
1819 | 1736 |
1820 static void DieFromDoubleFree() { | 1737 static void DieFromDoubleFree() { |
1821 char* p = NULL; | 1738 char* p = NULL; |
1822 p++; | 1739 p++; |
1823 *p += 1; // Segv. | 1740 *p += 1; // Segv. |
1824 } | 1741 } |
1825 | 1742 |
1826 static size_t DieFromBadFreePointer(void* unused) { | 1743 static size_t DieFromBadFreePointer(const void* unused) { |
1827 char* p = NULL; | 1744 char* p = NULL; |
1828 p += 2; | 1745 p += 2; |
1829 *p += 2; // Segv. | 1746 *p += 2; // Segv. |
1830 return 0; | 1747 return 0; |
1831 } | 1748 } |
1832 | 1749 |
1833 static void DieFromMemoryCorruption() { | 1750 static void DieFromMemoryCorruption() { |
1834 char* p = NULL; | 1751 char* p = NULL; |
1835 p += 3; | 1752 p += 3; |
1836 *p += 3; // Segv. | 1753 *p += 3; // Segv. |
(...skipping 99 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1936 *mark = ~allocated_mark; // Distinctively not allocated. | 1853 *mark = ~allocated_mark; // Distinctively not allocated. |
1937 } | 1854 } |
1938 | 1855 |
1939 static void MarkAllocatedRegion(void* ptr) { | 1856 static void MarkAllocatedRegion(void* ptr) { |
1940 if (ptr == NULL) return; | 1857 if (ptr == NULL) return; |
1941 MarkType* mark = GetMarkLocation(ptr); | 1858 MarkType* mark = GetMarkLocation(ptr); |
1942 *mark = GetMarkValue(ptr, mark); | 1859 *mark = GetMarkValue(ptr, mark); |
1943 } | 1860 } |
1944 | 1861 |
1945 #endif // TCMALLOC_VALIDATION | 1862 #endif // TCMALLOC_VALIDATION |
OLD | NEW |