OLD | NEW |
1 // Copyright (c) 2005, Google Inc. | 1 // Copyright (c) 2005, Google Inc. |
2 // All rights reserved. | 2 // All rights reserved. |
3 // | 3 // |
4 // Redistribution and use in source and binary forms, with or without | 4 // Redistribution and use in source and binary forms, with or without |
5 // modification, are permitted provided that the following conditions are | 5 // modification, are permitted provided that the following conditions are |
6 // met: | 6 // met: |
7 // | 7 // |
8 // * Redistributions of source code must retain the above copyright | 8 // * Redistributions of source code must retain the above copyright |
9 // notice, this list of conditions and the following disclaimer. | 9 // notice, this list of conditions and the following disclaimer. |
10 // * Redistributions in binary form must reproduce the above | 10 // * Redistributions in binary form must reproduce the above |
(...skipping 69 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
80 // TODO: Bias reclamation to larger addresses | 80 // TODO: Bias reclamation to larger addresses |
81 // TODO: implement mallinfo/mallopt | 81 // TODO: implement mallinfo/mallopt |
82 // TODO: Better testing | 82 // TODO: Better testing |
83 // | 83 // |
84 // 9/28/2003 (new page-level allocator replaces ptmalloc2): | 84 // 9/28/2003 (new page-level allocator replaces ptmalloc2): |
85 // * malloc/free of small objects goes from ~300 ns to ~50 ns. | 85 // * malloc/free of small objects goes from ~300 ns to ~50 ns. |
86 // * allocation of a reasonably complicated struct | 86 // * allocation of a reasonably complicated struct |
87 // goes from about 1100 ns to about 300 ns. | 87 // goes from about 1100 ns to about 300 ns. |
88 | 88 |
89 #include "config.h" | 89 #include "config.h" |
90 #include <gperftools/tcmalloc.h> | 90 #include <google/tcmalloc.h> |
91 | 91 |
92 #include <errno.h> // for ENOMEM, EINVAL, errno | 92 #include <errno.h> // for ENOMEM, EINVAL, errno |
93 #ifdef HAVE_SYS_CDEFS_H | 93 #ifdef HAVE_SYS_CDEFS_H |
94 #include <sys/cdefs.h> // for __THROW | 94 #include <sys/cdefs.h> // for __THROW |
95 #endif | 95 #endif |
| 96 #ifdef HAVE_FEATURES_H |
| 97 #include <features.h> // for __GLIBC__ |
| 98 #endif |
96 #if defined HAVE_STDINT_H | 99 #if defined HAVE_STDINT_H |
97 #include <stdint.h> | 100 #include <stdint.h> |
98 #elif defined HAVE_INTTYPES_H | 101 #elif defined HAVE_INTTYPES_H |
99 #include <inttypes.h> | 102 #include <inttypes.h> |
100 #else | 103 #else |
101 #include <sys/types.h> | 104 #include <sys/types.h> |
102 #endif | 105 #endif |
103 #include <stddef.h> // for size_t, NULL | 106 #include <stddef.h> // for size_t, NULL |
104 #include <stdlib.h> // for getenv | 107 #include <stdlib.h> // for getenv |
105 #include <string.h> // for strcmp, memset, strlen, etc | 108 #include <string.h> // for strcmp, memset, strlen, etc |
106 #ifdef HAVE_UNISTD_H | 109 #ifdef HAVE_UNISTD_H |
107 #include <unistd.h> // for getpagesize, write, etc | 110 #include <unistd.h> // for getpagesize, write, etc |
108 #endif | 111 #endif |
109 #include <algorithm> // for max, min | 112 #include <algorithm> // for max, min |
110 #include <limits> // for numeric_limits | 113 #include <limits> // for numeric_limits |
111 #include <new> // for nothrow_t (ptr only), etc | 114 #include <new> // for nothrow_t (ptr only), etc |
112 #include <vector> // for vector | 115 #include <vector> // for vector |
113 | 116 |
114 #include <gperftools/malloc_extension.h> | 117 #include <google/malloc_extension.h> |
115 #include <gperftools/malloc_hook.h> // for MallocHook | 118 #include <google/malloc_hook.h> // for MallocHook |
116 #include "base/basictypes.h" // for int64 | 119 #include "base/basictypes.h" // for int64 |
117 #include "base/commandlineflags.h" // for RegisterFlagValidator, etc | 120 #include "base/commandlineflags.h" // for RegisterFlagValidator, etc |
118 #include "base/dynamic_annotations.h" // for RunningOnValgrind | 121 #include "base/dynamic_annotations.h" // for RunningOnValgrind |
119 #include "base/spinlock.h" // for SpinLockHolder | 122 #include "base/spinlock.h" // for SpinLockHolder |
120 #include "central_freelist.h" // for CentralFreeListPadded | 123 #include "central_freelist.h" // for CentralFreeListPadded |
121 #include "common.h" // for StackTrace, kPageShift, etc | 124 #include "common.h" // for StackTrace, kPageShift, etc |
122 #include "free_list.h" // for FL_Init | 125 #include "free_list.h" // for FL_Init |
123 #include "internal_logging.h" // for ASSERT, TCMalloc_Printer, etc | 126 #include "internal_logging.h" // for ASSERT, TCMalloc_Printer, etc |
124 #include "malloc_hook-inl.h" // for MallocHook::InvokeNewHook, etc | 127 #include "malloc_hook-inl.h" // for MallocHook::InvokeNewHook, etc |
125 #include "page_heap.h" // for PageHeap, PageHeap::Stats | 128 #include "page_heap.h" // for PageHeap, PageHeap::Stats |
(...skipping 14 matching lines...) Expand all Loading... |
140 # include <sys/malloc.h> | 143 # include <sys/malloc.h> |
141 # elif defined(HAVE_MALLOC_MALLOC_H) | 144 # elif defined(HAVE_MALLOC_MALLOC_H) |
142 # include <malloc/malloc.h> | 145 # include <malloc/malloc.h> |
143 # endif | 146 # endif |
144 #endif | 147 #endif |
145 | 148 |
146 #if (defined(_WIN32) && !defined(__CYGWIN__) && !defined(__CYGWIN32__)) && !defi
ned(WIN32_OVERRIDE_ALLOCATORS) | 149 #if (defined(_WIN32) && !defined(__CYGWIN__) && !defined(__CYGWIN32__)) && !defi
ned(WIN32_OVERRIDE_ALLOCATORS) |
147 # define WIN32_DO_PATCHING 1 | 150 # define WIN32_DO_PATCHING 1 |
148 #endif | 151 #endif |
149 | 152 |
150 // Some windows file somewhere (at least on cygwin) #define's small (!) | 153 // GLibc 2.14+ requires the hook functions be declared volatile, based on the |
151 // For instance, <windows.h> appears to have "#define small char". | 154 // value of the define __MALLOC_HOOK_VOLATILE. For compatibility with |
152 #undef small | 155 // older/non-GLibc implementations, provide an empty definition. |
| 156 #if !defined(__MALLOC_HOOK_VOLATILE) |
| 157 #define __MALLOC_HOOK_VOLATILE |
| 158 #endif |
153 | 159 |
154 using STL_NAMESPACE::max; | 160 using STL_NAMESPACE::max; |
155 using STL_NAMESPACE::numeric_limits; | 161 using STL_NAMESPACE::numeric_limits; |
156 using STL_NAMESPACE::vector; | 162 using STL_NAMESPACE::vector; |
157 | |
158 #include "libc_override.h" | |
159 | |
160 // __THROW is defined in glibc (via <sys/cdefs.h>). It means, | |
161 // counter-intuitively, "This function will never throw an exception." | |
162 // It's an optional optimization tool, but we may need to use it to | |
163 // match glibc prototypes. | |
164 #ifndef __THROW // I guess we're not on a glibc system | |
165 # define __THROW // __THROW is just an optimization, so ok to make it "" | |
166 #endif | |
167 | |
168 using tcmalloc::AlignmentForSize; | 163 using tcmalloc::AlignmentForSize; |
169 using tcmalloc::kLog; | |
170 using tcmalloc::kCrash; | |
171 using tcmalloc::kCrashWithStats; | |
172 using tcmalloc::Log; | |
173 using tcmalloc::PageHeap; | 164 using tcmalloc::PageHeap; |
174 using tcmalloc::PageHeapAllocator; | 165 using tcmalloc::PageHeapAllocator; |
175 using tcmalloc::SizeMap; | 166 using tcmalloc::SizeMap; |
176 using tcmalloc::Span; | 167 using tcmalloc::Span; |
177 using tcmalloc::StackTrace; | 168 using tcmalloc::StackTrace; |
178 using tcmalloc::Static; | 169 using tcmalloc::Static; |
179 using tcmalloc::ThreadCache; | 170 using tcmalloc::ThreadCache; |
180 | 171 |
| 172 // __THROW is defined in glibc systems. It means, counter-intuitively, |
| 173 // "This function will never throw an exception." It's an optional |
| 174 // optimization tool, but we may need to use it to match glibc prototypes. |
| 175 #ifndef __THROW // I guess we're not on a glibc system |
| 176 # define __THROW // __THROW is just an optimization, so ok to make it "" |
| 177 #endif |
| 178 |
181 // ---- Double free debug declarations | 179 // ---- Double free debug declarations |
182 static size_t ExcludeSpaceForMark(size_t size); | 180 static size_t ExcludeSpaceForMark(size_t size); |
183 static void AddRoomForMark(size_t* size); | 181 static void AddRoomForMark(size_t* size); |
184 static void ExcludeMarkFromSize(size_t* new_size); | 182 static void ExcludeMarkFromSize(size_t* new_size); |
185 static void MarkAllocatedRegion(void* ptr); | 183 static void MarkAllocatedRegion(void* ptr); |
186 static void ValidateAllocatedRegion(void* ptr, size_t cl); | 184 static void ValidateAllocatedRegion(void* ptr, size_t cl); |
187 // ---- End Double free debug declarations | 185 // ---- End Double free debug declarations |
188 | 186 |
189 DECLARE_int64(tcmalloc_sample_parameter); | 187 DECLARE_int64(tcmalloc_sample_parameter); |
190 DECLARE_double(tcmalloc_release_rate); | 188 DECLARE_double(tcmalloc_release_rate); |
(...skipping 84 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
275 // Some non-standard extensions that we support. | 273 // Some non-standard extensions that we support. |
276 | 274 |
277 // This is equivalent to | 275 // This is equivalent to |
278 // OS X: malloc_size() | 276 // OS X: malloc_size() |
279 // glibc: malloc_usable_size() | 277 // glibc: malloc_usable_size() |
280 // Windows: _msize() | 278 // Windows: _msize() |
281 size_t tc_malloc_size(void* p) __THROW | 279 size_t tc_malloc_size(void* p) __THROW |
282 ATTRIBUTE_SECTION(google_malloc); | 280 ATTRIBUTE_SECTION(google_malloc); |
283 } // extern "C" | 281 } // extern "C" |
284 | 282 |
| 283 // Override the libc functions to prefer our own instead. This comes |
| 284 // first so code in tcmalloc.cc can use the overridden versions. One |
| 285 // exception: in windows, by default, we patch our code into these |
| 286 // functions (via src/windows/patch_function.cc) rather than override |
| 287 // them. In that case, we don't want to do this overriding here. |
| 288 #if !defined(WIN32_DO_PATCHING) |
| 289 |
| 290 // TODO(mbelshe): Turn off TCMalloc's symbols for libc. We do that |
| 291 // elsewhere. |
| 292 #ifndef _WIN32 |
| 293 |
| 294 #if defined(__GNUC__) && !defined(__MACH__) |
| 295 // Potentially faster variants that use the gcc alias extension. |
| 296 // FreeBSD does support aliases, but apparently not correctly. :-( |
| 297 // NOTE: we make many of these symbols weak, but do so in the makefile |
| 298 // (via objcopy -W) and not here. That ends up being more portable. |
| 299 # define ALIAS(x) __attribute__ ((alias (x))) |
| 300 void* operator new(size_t size) throw (std::bad_alloc) ALIAS("tc_new"); |
| 301 void operator delete(void* p) __THROW ALIAS("tc_delete"); |
| 302 void* operator new[](size_t size) throw (std::bad_alloc) ALIAS("tc_newarray"); |
| 303 void operator delete[](void* p) __THROW ALIAS("tc_deletearray"); |
| 304 void* operator new(size_t size, const std::nothrow_t&) __THROW |
| 305 ALIAS("tc_new_nothrow"); |
| 306 void* operator new[](size_t size, const std::nothrow_t&) __THROW |
| 307 ALIAS("tc_newarray_nothrow"); |
| 308 void operator delete(void* size, const std::nothrow_t&) __THROW |
| 309 ALIAS("tc_delete_nothrow"); |
| 310 void operator delete[](void* size, const std::nothrow_t&) __THROW |
| 311 ALIAS("tc_deletearray_nothrow"); |
| 312 extern "C" { |
| 313 void* malloc(size_t size) __THROW ALIAS("tc_malloc"); |
| 314 void free(void* ptr) __THROW ALIAS("tc_free"); |
| 315 void* realloc(void* ptr, size_t size) __THROW ALIAS("tc_realloc"); |
| 316 void* calloc(size_t n, size_t size) __THROW ALIAS("tc_calloc"); |
| 317 void cfree(void* ptr) __THROW ALIAS("tc_cfree"); |
| 318 void* memalign(size_t align, size_t s) __THROW ALIAS("tc_memalign"); |
| 319 void* valloc(size_t size) __THROW ALIAS("tc_valloc"); |
| 320 void* pvalloc(size_t size) __THROW ALIAS("tc_pvalloc"); |
| 321 int posix_memalign(void** r, size_t a, size_t s) __THROW |
| 322 ALIAS("tc_posix_memalign"); |
| 323 void malloc_stats(void) __THROW ALIAS("tc_malloc_stats"); |
| 324 int mallopt(int cmd, int value) __THROW ALIAS("tc_mallopt"); |
| 325 #ifdef HAVE_STRUCT_MALLINFO |
| 326 struct mallinfo mallinfo(void) __THROW ALIAS("tc_mallinfo"); |
| 327 #endif |
| 328 size_t malloc_size(void* p) __THROW ALIAS("tc_malloc_size"); |
| 329 size_t malloc_usable_size(void* p) __THROW ALIAS("tc_malloc_size"); |
| 330 } // extern "C" |
| 331 #else // #if defined(__GNUC__) && !defined(__MACH__) |
| 332 // Portable wrappers |
| 333 void* operator new(size_t size) { return tc_new(size); } |
| 334 void operator delete(void* p) __THROW { tc_delete(p); } |
| 335 void* operator new[](size_t size) { return tc_newarray(size); } |
| 336 void operator delete[](void* p) __THROW { tc_deletearray(p); } |
| 337 void* operator new(size_t size, const std::nothrow_t& nt) __THROW { |
| 338 return tc_new_nothrow(size, nt); |
| 339 } |
| 340 void* operator new[](size_t size, const std::nothrow_t& nt) __THROW { |
| 341 return tc_newarray_nothrow(size, nt); |
| 342 } |
| 343 void operator delete(void* ptr, const std::nothrow_t& nt) __THROW { |
| 344 return tc_delete_nothrow(ptr, nt); |
| 345 } |
| 346 void operator delete[](void* ptr, const std::nothrow_t& nt) __THROW { |
| 347 return tc_deletearray_nothrow(ptr, nt); |
| 348 } |
| 349 extern "C" { |
| 350 void* malloc(size_t s) __THROW { return tc_malloc(s); } |
| 351 void free(void* p) __THROW { tc_free(p); } |
| 352 void* realloc(void* p, size_t s) __THROW { return tc_realloc(p, s); } |
| 353 void* calloc(size_t n, size_t s) __THROW { return tc_calloc(n, s); } |
| 354 void cfree(void* p) __THROW { tc_cfree(p); } |
| 355 void* memalign(size_t a, size_t s) __THROW { return tc_memalign(a, s); } |
| 356 void* valloc(size_t s) __THROW { return tc_valloc(s); } |
| 357 void* pvalloc(size_t s) __THROW { return tc_pvalloc(s); } |
| 358 int posix_memalign(void** r, size_t a, size_t s) __THROW { |
| 359 return tc_posix_memalign(r, a, s); |
| 360 } |
| 361 void malloc_stats(void) __THROW { tc_malloc_stats(); } |
| 362 int mallopt(int cmd, int v) __THROW { return tc_mallopt(cmd, v); } |
| 363 #ifdef HAVE_STRUCT_MALLINFO |
| 364 struct mallinfo mallinfo(void) __THROW { return tc_mallinfo(); } |
| 365 #endif |
| 366 size_t malloc_size(void* p) __THROW { return tc_malloc_size(p); } |
| 367 size_t malloc_usable_size(void* p) __THROW { return tc_malloc_size(p); } |
| 368 } // extern "C" |
| 369 #endif // #if defined(__GNUC__) |
| 370 |
| 371 // Some library routines on RedHat 9 allocate memory using malloc() |
| 372 // and free it using __libc_free() (or vice-versa). Since we provide |
| 373 // our own implementations of malloc/free, we need to make sure that |
| 374 // the __libc_XXX variants (defined as part of glibc) also point to |
| 375 // the same implementations. |
| 376 #ifdef __GLIBC__ // only glibc defines __libc_* |
| 377 extern "C" { |
| 378 #ifdef ALIAS |
| 379 void* __libc_malloc(size_t size) ALIAS("tc_malloc"); |
| 380 void __libc_free(void* ptr) ALIAS("tc_free"); |
| 381 void* __libc_realloc(void* ptr, size_t size) ALIAS("tc_realloc"); |
| 382 void* __libc_calloc(size_t n, size_t size) ALIAS("tc_calloc"); |
| 383 void __libc_cfree(void* ptr) ALIAS("tc_cfree"); |
| 384 void* __libc_memalign(size_t align, size_t s) ALIAS("tc_memalign"); |
| 385 void* __libc_valloc(size_t size) ALIAS("tc_valloc"); |
| 386 void* __libc_pvalloc(size_t size) ALIAS("tc_pvalloc"); |
| 387 int __posix_memalign(void** r, size_t a, size_t s) ALIAS("tc_posix_memalign"); |
| 388 #else // #ifdef ALIAS |
| 389 void* __libc_malloc(size_t size) { return malloc(size); } |
| 390 void __libc_free(void* ptr) { free(ptr); } |
| 391 void* __libc_realloc(void* ptr, size_t size) { return realloc(ptr, size); } |
| 392 void* __libc_calloc(size_t n, size_t size) { return calloc(n, size); } |
| 393 void __libc_cfree(void* ptr) { cfree(ptr); } |
| 394 void* __libc_memalign(size_t align, size_t s) { return memalign(align, s); } |
| 395 void* __libc_valloc(size_t size) { return valloc(size); } |
| 396 void* __libc_pvalloc(size_t size) { return pvalloc(size); } |
| 397 int __posix_memalign(void** r, size_t a, size_t s) { |
| 398 return posix_memalign(r, a, s); |
| 399 } |
| 400 #endif // #ifdef ALIAS |
| 401 } // extern "C" |
| 402 #endif // ifdef __GLIBC__ |
| 403 |
| 404 #if defined(__GLIBC__) && defined(HAVE_MALLOC_H) |
| 405 // If we're using glibc, then override glibc malloc hooks to make sure that even |
| 406 // if calls fall through to ptmalloc (due to dlopen() with RTLD_DEEPBIND or what |
| 407 // not), ptmalloc will use TCMalloc. |
| 408 |
| 409 static void* tc_ptmalloc_malloc_hook(size_t size, const void* caller) { |
| 410 return tc_malloc(size); |
| 411 } |
| 412 |
| 413 void* (*__MALLOC_HOOK_VOLATILE __malloc_hook)( |
| 414 size_t size, const void* caller) = tc_ptmalloc_malloc_hook; |
| 415 |
| 416 static void* tc_ptmalloc_realloc_hook( |
| 417 void* ptr, size_t size, const void* caller) { |
| 418 return tc_realloc(ptr, size); |
| 419 } |
| 420 |
| 421 void* (*__MALLOC_HOOK_VOLATILE __realloc_hook)( |
| 422 void* ptr, size_t size, const void* caller) = tc_ptmalloc_realloc_hook; |
| 423 |
| 424 static void tc_ptmalloc_free_hook(void* ptr, const void* caller) { |
| 425 tc_free(ptr); |
| 426 } |
| 427 |
| 428 void (*__MALLOC_HOOK_VOLATILE __free_hook)(void* ptr, const void* caller) = tc_p
tmalloc_free_hook; |
| 429 |
| 430 #endif |
| 431 |
| 432 #endif // #ifndef _WIN32 |
| 433 #undef ALIAS |
| 434 |
| 435 #endif // #ifndef(WIN32_DO_PATCHING) |
| 436 |
285 | 437 |
286 // ----------------------- IMPLEMENTATION ------------------------------- | 438 // ----------------------- IMPLEMENTATION ------------------------------- |
287 | 439 |
288 static int tc_new_mode = 0; // See tc_set_new_mode(). | 440 static int tc_new_mode = 0; // See tc_set_new_mode(). |
289 | 441 |
290 // Routines such as free() and realloc() catch some erroneous pointers | 442 // Routines such as free() and realloc() catch some erroneous pointers |
291 // passed to them, and invoke the below when they do. (An erroneous pointer | 443 // passed to them, and invoke the below when they do. (An erroneous pointer |
292 // won't be caught if it's within a valid span or a stale span for which | 444 // won't be caught if it's within a valid span or a stale span for which |
293 // the pagemap cache has a non-zero sizeclass.) This is a cheap (source-editing | 445 // the pagemap cache has a non-zero sizeclass.) This is a cheap (source-editing |
294 // required) kind of exception handling for these routines. | 446 // required) kind of exception handling for these routines. |
295 namespace { | 447 namespace { |
296 void InvalidFree(void* ptr) { | 448 void InvalidFree(void* ptr) { |
297 Log(kCrash, __FILE__, __LINE__, "Attempt to free invalid pointer", ptr); | 449 CRASH("Attempt to free invalid pointer: %p\n", ptr); |
298 } | 450 } |
299 | 451 |
300 size_t InvalidGetSizeForRealloc(const void* old_ptr) { | 452 size_t InvalidGetSizeForRealloc(void* old_ptr) { |
301 Log(kCrash, __FILE__, __LINE__, | 453 CRASH("Attempt to realloc invalid pointer: %p\n", old_ptr); |
302 "Attempt to realloc invalid pointer", old_ptr); | |
303 return 0; | 454 return 0; |
304 } | 455 } |
305 | 456 |
306 size_t InvalidGetAllocatedSize(const void* ptr) { | 457 size_t InvalidGetAllocatedSize(void* ptr) { |
307 Log(kCrash, __FILE__, __LINE__, | 458 CRASH("Attempt to get the size of an invalid pointer: %p\n", ptr); |
308 "Attempt to get the size of an invalid pointer", ptr); | |
309 return 0; | 459 return 0; |
310 } | 460 } |
311 } // unnamed namespace | 461 } // unnamed namespace |
312 | 462 |
313 // Extract interesting stats | 463 // Extract interesting stats |
314 struct TCMallocStats { | 464 struct TCMallocStats { |
315 uint64_t thread_bytes; // Bytes in thread caches | 465 uint64_t thread_bytes; // Bytes in thread caches |
316 uint64_t central_bytes; // Bytes in central cache | 466 uint64_t central_bytes; // Bytes in central cache |
317 uint64_t transfer_bytes; // Bytes in central transfer cache | 467 uint64_t transfer_bytes; // Bytes in central transfer cache |
318 uint64_t metadata_bytes; // Bytes alloced for metadata | 468 uint64_t metadata_bytes; // Bytes alloced for metadata |
319 PageHeap::Stats pageheap; // Stats from page heap | 469 PageHeap::Stats pageheap; // Stats from page heap |
320 }; | 470 }; |
321 | 471 |
322 // Get stats into "r". Also get per-size-class counts if class_count != NULL | 472 // Get stats into "r". Also get per-size-class counts if class_count != NULL |
323 static void ExtractStats(TCMallocStats* r, uint64_t* class_count, | 473 static void ExtractStats(TCMallocStats* r, uint64_t* class_count) { |
324 PageHeap::SmallSpanStats* small_spans, | |
325 PageHeap::LargeSpanStats* large_spans) { | |
326 r->central_bytes = 0; | 474 r->central_bytes = 0; |
327 r->transfer_bytes = 0; | 475 r->transfer_bytes = 0; |
328 for (int cl = 0; cl < kNumClasses; ++cl) { | 476 for (int cl = 0; cl < kNumClasses; ++cl) { |
329 const int length = Static::central_cache()[cl].length(); | 477 const int length = Static::central_cache()[cl].length(); |
330 const int tc_length = Static::central_cache()[cl].tc_length(); | 478 const int tc_length = Static::central_cache()[cl].tc_length(); |
331 const size_t cache_overhead = Static::central_cache()[cl].OverheadBytes(); | |
332 const size_t size = static_cast<uint64_t>( | 479 const size_t size = static_cast<uint64_t>( |
333 Static::sizemap()->ByteSizeForClass(cl)); | 480 Static::sizemap()->ByteSizeForClass(cl)); |
334 r->central_bytes += (size * length) + cache_overhead; | 481 r->central_bytes += (size * length); |
335 r->transfer_bytes += (size * tc_length); | 482 r->transfer_bytes += (size * tc_length); |
336 if (class_count) class_count[cl] = length + tc_length; | 483 if (class_count) class_count[cl] = length + tc_length; |
337 } | 484 } |
338 | 485 |
339 // Add stats from per-thread heaps | 486 // Add stats from per-thread heaps |
340 r->thread_bytes = 0; | 487 r->thread_bytes = 0; |
341 { // scope | 488 { // scope |
342 SpinLockHolder h(Static::pageheap_lock()); | 489 SpinLockHolder h(Static::pageheap_lock()); |
343 ThreadCache::GetThreadStats(&r->thread_bytes, class_count); | 490 ThreadCache::GetThreadStats(&r->thread_bytes, class_count); |
344 r->metadata_bytes = tcmalloc::metadata_system_bytes(); | 491 r->metadata_bytes = tcmalloc::metadata_system_bytes(); |
345 r->pageheap = Static::pageheap()->stats(); | 492 r->pageheap = Static::pageheap()->stats(); |
346 if (small_spans != NULL) { | |
347 Static::pageheap()->GetSmallSpanStats(small_spans); | |
348 } | |
349 if (large_spans != NULL) { | |
350 Static::pageheap()->GetLargeSpanStats(large_spans); | |
351 } | |
352 } | 493 } |
353 } | 494 } |
354 | 495 |
355 static double PagesToMiB(uint64_t pages) { | |
356 return (pages << kPageShift) / 1048576.0; | |
357 } | |
358 | |
359 // WRITE stats to "out" | 496 // WRITE stats to "out" |
360 static void DumpStats(TCMalloc_Printer* out, int level) { | 497 static void DumpStats(TCMalloc_Printer* out, int level) { |
361 TCMallocStats stats; | 498 TCMallocStats stats; |
362 uint64_t class_count[kNumClasses]; | 499 uint64_t class_count[kNumClasses]; |
363 PageHeap::SmallSpanStats small; | 500 ExtractStats(&stats, (level >= 2 ? class_count : NULL)); |
364 PageHeap::LargeSpanStats large; | |
365 if (level >= 2) { | |
366 ExtractStats(&stats, class_count, &small, &large); | |
367 } else { | |
368 ExtractStats(&stats, NULL, NULL, NULL); | |
369 } | |
370 | 501 |
371 static const double MiB = 1048576.0; | 502 static const double MiB = 1048576.0; |
372 | 503 |
373 const uint64_t virtual_memory_used = (stats.pageheap.system_bytes | 504 const uint64_t virtual_memory_used = (stats.pageheap.system_bytes |
374 + stats.metadata_bytes); | 505 + stats.metadata_bytes); |
375 const uint64_t physical_memory_used = (virtual_memory_used | 506 const uint64_t physical_memory_used = (virtual_memory_used |
376 - stats.pageheap.unmapped_bytes); | 507 - stats.pageheap.unmapped_bytes); |
377 const uint64_t bytes_in_use_by_app = (physical_memory_used | 508 const uint64_t bytes_in_use_by_app = (physical_memory_used |
378 - stats.metadata_bytes | 509 - stats.metadata_bytes |
379 - stats.pageheap.free_bytes | 510 - stats.pageheap.free_bytes |
(...skipping 63 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
443 cumulative += class_bytes; | 574 cumulative += class_bytes; |
444 out->printf("class %3d [ %8" PRIuS " bytes ] : " | 575 out->printf("class %3d [ %8" PRIuS " bytes ] : " |
445 "%8" PRIu64 " objs; %5.1f MiB; %5.1f cum MiB\n", | 576 "%8" PRIu64 " objs; %5.1f MiB; %5.1f cum MiB\n", |
446 cl, Static::sizemap()->ByteSizeForClass(cl), | 577 cl, Static::sizemap()->ByteSizeForClass(cl), |
447 class_count[cl], | 578 class_count[cl], |
448 class_bytes / MiB, | 579 class_bytes / MiB, |
449 cumulative / MiB); | 580 cumulative / MiB); |
450 } | 581 } |
451 } | 582 } |
452 | 583 |
453 // append page heap info | 584 SpinLockHolder h(Static::pageheap_lock()); |
454 int nonempty_sizes = 0; | 585 Static::pageheap()->Dump(out); |
455 for (int s = 0; s < kMaxPages; s++) { | |
456 if (small.normal_length[s] + small.returned_length[s] > 0) { | |
457 nonempty_sizes++; | |
458 } | |
459 } | |
460 out->printf("------------------------------------------------\n"); | |
461 out->printf("PageHeap: %d sizes; %6.1f MiB free; %6.1f MiB unmapped\n", | |
462 nonempty_sizes, stats.pageheap.free_bytes / MiB, | |
463 stats.pageheap.unmapped_bytes / MiB); | |
464 out->printf("------------------------------------------------\n"); | |
465 uint64_t total_normal = 0; | |
466 uint64_t total_returned = 0; | |
467 for (int s = 0; s < kMaxPages; s++) { | |
468 const int n_length = small.normal_length[s]; | |
469 const int r_length = small.returned_length[s]; | |
470 if (n_length + r_length > 0) { | |
471 uint64_t n_pages = s * n_length; | |
472 uint64_t r_pages = s * r_length; | |
473 total_normal += n_pages; | |
474 total_returned += r_pages; | |
475 out->printf("%6u pages * %6u spans ~ %6.1f MiB; %6.1f MiB cum" | |
476 "; unmapped: %6.1f MiB; %6.1f MiB cum\n", | |
477 s, | |
478 (n_length + r_length), | |
479 PagesToMiB(n_pages + r_pages), | |
480 PagesToMiB(total_normal + total_returned), | |
481 PagesToMiB(r_pages), | |
482 PagesToMiB(total_returned)); | |
483 } | |
484 } | |
485 | |
486 total_normal += large.normal_pages; | |
487 total_returned += large.returned_pages; | |
488 out->printf(">255 large * %6u spans ~ %6.1f MiB; %6.1f MiB cum" | |
489 "; unmapped: %6.1f MiB; %6.1f MiB cum\n", | |
490 static_cast<unsigned int>(large.spans), | |
491 PagesToMiB(large.normal_pages + large.returned_pages), | |
492 PagesToMiB(total_normal + total_returned), | |
493 PagesToMiB(large.returned_pages), | |
494 PagesToMiB(total_returned)); | |
495 } | 586 } |
496 } | 587 } |
497 | 588 |
498 static void PrintStats(int level) { | 589 static void PrintStats(int level) { |
499 const int kBufferSize = 16 << 10; | 590 const int kBufferSize = 16 << 10; |
500 char* buffer = new char[kBufferSize]; | 591 char* buffer = new char[kBufferSize]; |
501 TCMalloc_Printer printer(buffer, kBufferSize); | 592 TCMalloc_Printer printer(buffer, kBufferSize); |
502 DumpStats(&printer, level); | 593 DumpStats(&printer, level); |
503 write(STDERR_FILENO, buffer, strlen(buffer)); | 594 write(STDERR_FILENO, buffer, strlen(buffer)); |
504 delete[] buffer; | 595 delete[] buffer; |
505 } | 596 } |
506 | 597 |
507 static void** DumpHeapGrowthStackTraces() { | 598 static void** DumpHeapGrowthStackTraces() { |
508 // Count how much space we need | 599 // Count how much space we need |
509 int needed_slots = 0; | 600 int needed_slots = 0; |
510 { | 601 { |
511 SpinLockHolder h(Static::pageheap_lock()); | 602 SpinLockHolder h(Static::pageheap_lock()); |
512 for (StackTrace* t = Static::growth_stacks(); | 603 for (StackTrace* t = Static::growth_stacks(); |
513 t != NULL; | 604 t != NULL; |
514 t = reinterpret_cast<StackTrace*>( | 605 t = reinterpret_cast<StackTrace*>( |
515 t->stack[tcmalloc::kMaxStackDepth-1])) { | 606 t->stack[tcmalloc::kMaxStackDepth-1])) { |
516 needed_slots += 3 + t->depth; | 607 needed_slots += 3 + t->depth; |
517 } | 608 } |
518 needed_slots += 100; // Slop in case list grows | 609 needed_slots += 100; // Slop in case list grows |
519 needed_slots += needed_slots/8; // An extra 12.5% slop | 610 needed_slots += needed_slots/8; // An extra 12.5% slop |
520 } | 611 } |
521 | 612 |
522 void** result = new void*[needed_slots]; | 613 void** result = new void*[needed_slots]; |
523 if (result == NULL) { | 614 if (result == NULL) { |
524 Log(kLog, __FILE__, __LINE__, | 615 MESSAGE("tcmalloc: allocation failed for stack trace slots", |
525 "tcmalloc: allocation failed for stack trace slots", | 616 needed_slots * sizeof(*result)); |
526 needed_slots * sizeof(*result)); | |
527 return NULL; | 617 return NULL; |
528 } | 618 } |
529 | 619 |
530 SpinLockHolder h(Static::pageheap_lock()); | 620 SpinLockHolder h(Static::pageheap_lock()); |
531 int used_slots = 0; | 621 int used_slots = 0; |
532 for (StackTrace* t = Static::growth_stacks(); | 622 for (StackTrace* t = Static::growth_stacks(); |
533 t != NULL; | 623 t != NULL; |
534 t = reinterpret_cast<StackTrace*>( | 624 t = reinterpret_cast<StackTrace*>( |
535 t->stack[tcmalloc::kMaxStackDepth-1])) { | 625 t->stack[tcmalloc::kMaxStackDepth-1])) { |
536 ASSERT(used_slots < needed_slots); // Need to leave room for terminator | 626 ASSERT(used_slots < needed_slots); // Need to leave room for terminator |
(...skipping 105 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
642 | 732 |
643 virtual void Ranges(void* arg, RangeFunction func) { | 733 virtual void Ranges(void* arg, RangeFunction func) { |
644 IterateOverRanges(arg, func); | 734 IterateOverRanges(arg, func); |
645 } | 735 } |
646 | 736 |
647 virtual bool GetNumericProperty(const char* name, size_t* value) { | 737 virtual bool GetNumericProperty(const char* name, size_t* value) { |
648 ASSERT(name != NULL); | 738 ASSERT(name != NULL); |
649 | 739 |
650 if (strcmp(name, "generic.current_allocated_bytes") == 0) { | 740 if (strcmp(name, "generic.current_allocated_bytes") == 0) { |
651 TCMallocStats stats; | 741 TCMallocStats stats; |
652 ExtractStats(&stats, NULL, NULL, NULL); | 742 ExtractStats(&stats, NULL); |
653 *value = stats.pageheap.system_bytes | 743 *value = stats.pageheap.system_bytes |
654 - stats.thread_bytes | 744 - stats.thread_bytes |
655 - stats.central_bytes | 745 - stats.central_bytes |
656 - stats.transfer_bytes | 746 - stats.transfer_bytes |
657 - stats.pageheap.free_bytes | 747 - stats.pageheap.free_bytes |
658 - stats.pageheap.unmapped_bytes; | 748 - stats.pageheap.unmapped_bytes; |
659 return true; | 749 return true; |
660 } | 750 } |
661 | 751 |
662 if (strcmp(name, "generic.heap_size") == 0) { | 752 if (strcmp(name, "generic.heap_size") == 0) { |
663 TCMallocStats stats; | 753 TCMallocStats stats; |
664 ExtractStats(&stats, NULL, NULL, NULL); | 754 ExtractStats(&stats, NULL); |
665 *value = stats.pageheap.system_bytes; | 755 *value = stats.pageheap.system_bytes; |
666 return true; | 756 return true; |
667 } | 757 } |
668 | 758 |
669 if (strcmp(name, "tcmalloc.slack_bytes") == 0) { | 759 if (strcmp(name, "tcmalloc.slack_bytes") == 0) { |
670 // Kept for backwards compatibility. Now defined externally as: | 760 // Kept for backwards compatibility. Now defined externally as: |
671 // pageheap_free_bytes + pageheap_unmapped_bytes. | 761 // pageheap_free_bytes + pageheap_unmapped_bytes. |
672 SpinLockHolder l(Static::pageheap_lock()); | 762 SpinLockHolder l(Static::pageheap_lock()); |
673 PageHeap::Stats stats = Static::pageheap()->stats(); | 763 PageHeap::Stats stats = Static::pageheap()->stats(); |
674 *value = stats.free_bytes + stats.unmapped_bytes; | 764 *value = stats.free_bytes + stats.unmapped_bytes; |
(...skipping 13 matching lines...) Expand all Loading... |
688 } | 778 } |
689 | 779 |
690 if (strcmp(name, "tcmalloc.max_total_thread_cache_bytes") == 0) { | 780 if (strcmp(name, "tcmalloc.max_total_thread_cache_bytes") == 0) { |
691 SpinLockHolder l(Static::pageheap_lock()); | 781 SpinLockHolder l(Static::pageheap_lock()); |
692 *value = ThreadCache::overall_thread_cache_size(); | 782 *value = ThreadCache::overall_thread_cache_size(); |
693 return true; | 783 return true; |
694 } | 784 } |
695 | 785 |
696 if (strcmp(name, "tcmalloc.current_total_thread_cache_bytes") == 0) { | 786 if (strcmp(name, "tcmalloc.current_total_thread_cache_bytes") == 0) { |
697 TCMallocStats stats; | 787 TCMallocStats stats; |
698 ExtractStats(&stats, NULL, NULL, NULL); | 788 ExtractStats(&stats, NULL); |
699 *value = stats.thread_bytes; | 789 *value = stats.thread_bytes; |
700 return true; | 790 return true; |
701 } | 791 } |
702 | 792 |
703 return false; | 793 return false; |
704 } | 794 } |
705 | 795 |
706 virtual bool SetNumericProperty(const char* name, size_t value) { | 796 virtual bool SetNumericProperty(const char* name, size_t value) { |
707 ASSERT(name != NULL); | 797 ASSERT(name != NULL); |
708 | 798 |
(...skipping 60 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
769 const size_t alloc_size = Static::sizemap()->ByteSizeForClass(cl); | 859 const size_t alloc_size = Static::sizemap()->ByteSizeForClass(cl); |
770 return alloc_size; | 860 return alloc_size; |
771 } else { | 861 } else { |
772 return tcmalloc::pages(size) << kPageShift; | 862 return tcmalloc::pages(size) << kPageShift; |
773 } | 863 } |
774 } | 864 } |
775 | 865 |
776 // This just calls GetSizeWithCallback, but because that's in an | 866 // This just calls GetSizeWithCallback, but because that's in an |
777 // unnamed namespace, we need to move the definition below it in the | 867 // unnamed namespace, we need to move the definition below it in the |
778 // file. | 868 // file. |
779 virtual size_t GetAllocatedSize(const void* ptr); | 869 virtual size_t GetAllocatedSize(void* ptr); |
780 | |
781 // This duplicates some of the logic in GetSizeWithCallback, but is | |
782 // faster. This is important on OS X, where this function is called | |
783 // on every allocation operation. | |
784 virtual Ownership GetOwnership(const void* ptr) { | |
785 const PageID p = reinterpret_cast<uintptr_t>(ptr) >> kPageShift; | |
786 // The rest of tcmalloc assumes that all allocated pointers use at | |
787 // most kAddressBits bits. If ptr doesn't, then it definitely | |
788 // wasn't alloacted by tcmalloc. | |
789 if ((p >> (kAddressBits - kPageShift)) > 0) { | |
790 return kNotOwned; | |
791 } | |
792 size_t cl = Static::pageheap()->GetSizeClassIfCached(p); | |
793 if (cl != 0) { | |
794 return kOwned; | |
795 } | |
796 const Span *span = Static::pageheap()->GetDescriptor(p); | |
797 return span ? kOwned : kNotOwned; | |
798 } | |
799 | 870 |
800 virtual void GetFreeListSizes(vector<MallocExtension::FreeListInfo>* v) { | 871 virtual void GetFreeListSizes(vector<MallocExtension::FreeListInfo>* v) { |
801 static const char* kCentralCacheType = "tcmalloc.central"; | 872 static const char* kCentralCacheType = "tcmalloc.central"; |
802 static const char* kTransferCacheType = "tcmalloc.transfer"; | 873 static const char* kTransferCacheType = "tcmalloc.transfer"; |
803 static const char* kThreadCacheType = "tcmalloc.thread"; | 874 static const char* kThreadCacheType = "tcmalloc.thread"; |
804 static const char* kPageHeapType = "tcmalloc.page"; | 875 static const char* kPageHeapType = "tcmalloc.page"; |
805 static const char* kPageHeapUnmappedType = "tcmalloc.page_unmapped"; | 876 static const char* kPageHeapUnmappedType = "tcmalloc.page_unmapped"; |
806 static const char* kLargeSpanType = "tcmalloc.large"; | 877 static const char* kLargeSpanType = "tcmalloc.large"; |
807 static const char* kLargeUnmappedSpanType = "tcmalloc.large_unmapped"; | 878 static const char* kLargeUnmappedSpanType = "tcmalloc.large_unmapped"; |
808 | 879 |
(...skipping 34 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
843 MallocExtension::FreeListInfo i; | 914 MallocExtension::FreeListInfo i; |
844 i.min_object_size = prev_class_size + 1; | 915 i.min_object_size = prev_class_size + 1; |
845 i.max_object_size = Static::sizemap()->ByteSizeForClass(cl); | 916 i.max_object_size = Static::sizemap()->ByteSizeForClass(cl); |
846 i.total_bytes_free = | 917 i.total_bytes_free = |
847 class_count[cl] * Static::sizemap()->ByteSizeForClass(cl); | 918 class_count[cl] * Static::sizemap()->ByteSizeForClass(cl); |
848 i.type = kThreadCacheType; | 919 i.type = kThreadCacheType; |
849 v->push_back(i); | 920 v->push_back(i); |
850 } | 921 } |
851 | 922 |
852 // append page heap info | 923 // append page heap info |
853 PageHeap::SmallSpanStats small; | 924 int64 page_count_normal[kMaxPages]; |
854 PageHeap::LargeSpanStats large; | 925 int64 page_count_returned[kMaxPages]; |
| 926 int64 span_count_normal; |
| 927 int64 span_count_returned; |
855 { | 928 { |
856 SpinLockHolder h(Static::pageheap_lock()); | 929 SpinLockHolder h(Static::pageheap_lock()); |
857 Static::pageheap()->GetSmallSpanStats(&small); | 930 Static::pageheap()->GetClassSizes(page_count_normal, |
858 Static::pageheap()->GetLargeSpanStats(&large); | 931 page_count_returned, |
| 932 &span_count_normal, |
| 933 &span_count_returned); |
859 } | 934 } |
860 | 935 |
861 // large spans: mapped | 936 // spans: mapped |
862 MallocExtension::FreeListInfo span_info; | 937 MallocExtension::FreeListInfo span_info; |
863 span_info.type = kLargeSpanType; | 938 span_info.type = kLargeSpanType; |
864 span_info.max_object_size = (numeric_limits<size_t>::max)(); | 939 span_info.max_object_size = (numeric_limits<size_t>::max)(); |
865 span_info.min_object_size = kMaxPages << kPageShift; | 940 span_info.min_object_size = kMaxPages << kPageShift; |
866 span_info.total_bytes_free = large.normal_pages << kPageShift; | 941 span_info.total_bytes_free = span_count_normal << kPageShift; |
867 v->push_back(span_info); | 942 v->push_back(span_info); |
868 | 943 |
869 // large spans: unmapped | 944 // spans: unmapped |
870 span_info.type = kLargeUnmappedSpanType; | 945 span_info.type = kLargeUnmappedSpanType; |
871 span_info.total_bytes_free = large.returned_pages << kPageShift; | 946 span_info.total_bytes_free = span_count_returned << kPageShift; |
872 v->push_back(span_info); | 947 v->push_back(span_info); |
873 | 948 |
874 // small spans | |
875 for (int s = 1; s < kMaxPages; s++) { | 949 for (int s = 1; s < kMaxPages; s++) { |
876 MallocExtension::FreeListInfo i; | 950 MallocExtension::FreeListInfo i; |
877 i.max_object_size = (s << kPageShift); | 951 i.max_object_size = (s << kPageShift); |
878 i.min_object_size = ((s - 1) << kPageShift); | 952 i.min_object_size = ((s - 1) << kPageShift); |
879 | 953 |
880 i.type = kPageHeapType; | 954 i.type = kPageHeapType; |
881 i.total_bytes_free = (s << kPageShift) * small.normal_length[s]; | 955 i.total_bytes_free = (s << kPageShift) * page_count_normal[s]; |
882 v->push_back(i); | 956 v->push_back(i); |
883 | 957 |
884 i.type = kPageHeapUnmappedType; | 958 i.type = kPageHeapUnmappedType; |
885 i.total_bytes_free = (s << kPageShift) * small.returned_length[s]; | 959 i.total_bytes_free = (s << kPageShift) * page_count_returned[s]; |
886 v->push_back(i); | 960 v->push_back(i); |
887 } | 961 } |
888 } | 962 } |
889 }; | 963 }; |
890 | 964 |
891 // The constructor allocates an object to ensure that initialization | 965 // The constructor allocates an object to ensure that initialization |
892 // runs before main(), and therefore we do not have a chance to become | 966 // runs before main(), and therefore we do not have a chance to become |
893 // multi-threaded before initialization. We also create the TSD key | 967 // multi-threaded before initialization. We also create the TSD key |
894 // here. Presumably by the time this constructor runs, glibc is in | 968 // here. Presumably by the time this constructor runs, glibc is in |
895 // good enough shape to handle pthread_key_create(). | 969 // good enough shape to handle pthread_key_create(). |
896 // | 970 // |
897 // The constructor also takes the opportunity to tell STL to use | 971 // The constructor also takes the opportunity to tell STL to use |
898 // tcmalloc. We want to do this early, before construct time, so | 972 // tcmalloc. We want to do this early, before construct time, so |
899 // all user STL allocations go through tcmalloc (which works really | 973 // all user STL allocations go through tcmalloc (which works really |
900 // well for STL). | 974 // well for STL). |
901 // | 975 // |
902 // The destructor prints stats when the program exits. | 976 // The destructor prints stats when the program exits. |
903 static int tcmallocguard_refcount = 0; // no lock needed: runs before main() | 977 static int tcmallocguard_refcount = 0; // no lock needed: runs before main() |
904 TCMallocGuard::TCMallocGuard() { | 978 TCMallocGuard::TCMallocGuard() { |
905 if (tcmallocguard_refcount++ == 0) { | 979 if (tcmallocguard_refcount++ == 0) { |
906 #ifdef HAVE_TLS // this is true if the cc/ld/libc combo support TLS | 980 #ifdef HAVE_TLS // this is true if the cc/ld/libc combo support TLS |
907 // Check whether the kernel also supports TLS (needs to happen at runtime) | 981 // Check whether the kernel also supports TLS (needs to happen at runtime) |
908 tcmalloc::CheckIfKernelSupportsTLS(); | 982 tcmalloc::CheckIfKernelSupportsTLS(); |
909 #endif | 983 #endif |
910 ReplaceSystemAlloc(); // defined in libc_override_*.h | 984 #ifdef WIN32_DO_PATCHING |
| 985 // patch the windows VirtualAlloc, etc. |
| 986 PatchWindowsFunctions(); // defined in windows/patch_functions.cc |
| 987 #endif |
911 tc_free(tc_malloc(1)); | 988 tc_free(tc_malloc(1)); |
912 ThreadCache::InitTSD(); | 989 ThreadCache::InitTSD(); |
913 tc_free(tc_malloc(1)); | 990 tc_free(tc_malloc(1)); |
914 // Either we, or debugallocation.cc, or valgrind will control memory | 991 // Either we, or debugallocation.cc, or valgrind will control memory |
915 // management. We register our extension if we're the winner. | 992 // management. We register our extension if we're the winner. |
916 #ifdef TCMALLOC_USING_DEBUGALLOCATION | 993 #ifdef TCMALLOC_USING_DEBUGALLOCATION |
917 // Let debugallocation register its extension. | 994 // Let debugallocation register its extension. |
918 #else | 995 #else |
919 if (RunningOnValgrind()) { | 996 if (RunningOnValgrind()) { |
920 // Let Valgrind uses its own malloc (so don't register our extension). | 997 // Let Valgrind uses its own malloc (so don't register our extension). |
(...skipping 76 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
997 (kPageSize > FLAGS_tcmalloc_large_alloc_report_threshold | 1074 (kPageSize > FLAGS_tcmalloc_large_alloc_report_threshold |
998 ? kPageSize : FLAGS_tcmalloc_large_alloc_report_threshold); | 1075 ? kPageSize : FLAGS_tcmalloc_large_alloc_report_threshold); |
999 | 1076 |
1000 static void ReportLargeAlloc(Length num_pages, void* result) { | 1077 static void ReportLargeAlloc(Length num_pages, void* result) { |
1001 StackTrace stack; | 1078 StackTrace stack; |
1002 stack.depth = GetStackTrace(stack.stack, tcmalloc::kMaxStackDepth, 1); | 1079 stack.depth = GetStackTrace(stack.stack, tcmalloc::kMaxStackDepth, 1); |
1003 | 1080 |
1004 static const int N = 1000; | 1081 static const int N = 1000; |
1005 char buffer[N]; | 1082 char buffer[N]; |
1006 TCMalloc_Printer printer(buffer, N); | 1083 TCMalloc_Printer printer(buffer, N); |
1007 printer.printf("tcmalloc: large alloc %"PRIu64" bytes == %p @ ", | 1084 printer.printf("tcmalloc: large alloc %llu bytes == %p @ ", |
1008 static_cast<uint64>(num_pages) << kPageShift, | 1085 static_cast<unsigned long long>(num_pages) << kPageShift, |
1009 result); | 1086 result); |
1010 for (int i = 0; i < stack.depth; i++) { | 1087 for (int i = 0; i < stack.depth; i++) { |
1011 printer.printf(" %p", stack.stack[i]); | 1088 printer.printf(" %p", stack.stack[i]); |
1012 } | 1089 } |
1013 printer.printf("\n"); | 1090 printer.printf("\n"); |
1014 write(STDERR_FILENO, buffer, strlen(buffer)); | 1091 write(STDERR_FILENO, buffer, strlen(buffer)); |
1015 } | 1092 } |
1016 | 1093 |
1017 inline void* cpp_alloc(size_t size, bool nothrow); | 1094 inline void* cpp_alloc(size_t size, bool nothrow); |
1018 inline void* do_malloc(size_t size); | 1095 inline void* do_malloc(size_t size); |
(...skipping 68 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1087 // TODO(jar): If this has any detectable performance impact, it can be | 1164 // TODO(jar): If this has any detectable performance impact, it can be |
1088 // optimized by only tallying sizes if the profiler was activated to recall | 1165 // optimized by only tallying sizes if the profiler was activated to recall |
1089 // these tallies. I don't think this is performance critical, but we really | 1166 // these tallies. I don't think this is performance critical, but we really |
1090 // should measure it. | 1167 // should measure it. |
1091 heap->AddToByteAllocatedTotal(size); // Chromium profiling. | 1168 heap->AddToByteAllocatedTotal(size); // Chromium profiling. |
1092 | 1169 |
1093 if ((FLAGS_tcmalloc_sample_parameter > 0) && heap->SampleAllocation(size)) { | 1170 if ((FLAGS_tcmalloc_sample_parameter > 0) && heap->SampleAllocation(size)) { |
1094 ret = DoSampledAllocation(size); | 1171 ret = DoSampledAllocation(size); |
1095 MarkAllocatedRegion(ret); | 1172 MarkAllocatedRegion(ret); |
1096 } else { | 1173 } else { |
1097 // The common case, and also the simplest. This just pops the | 1174 // The common case, and also the simplest. This just pops the |
1098 // size-appropriate freelist, after replenishing it if it's empty. | 1175 // size-appropriate freelist, after replenishing it if it's empty. |
1099 ret = CheckedMallocResult(heap->Allocate(size, cl)); | 1176 ret = CheckedMallocResult(heap->Allocate(size, cl)); |
1100 } | 1177 } |
1101 } else { | 1178 } else { |
1102 ret = do_malloc_pages(heap, size); | 1179 ret = do_malloc_pages(heap, size); |
1103 MarkAllocatedRegion(ret); | 1180 MarkAllocatedRegion(ret); |
1104 } | 1181 } |
1105 if (ret == NULL) errno = ENOMEM; | 1182 if (ret == NULL) errno = ENOMEM; |
1106 return ret; | 1183 return ret; |
1107 } | 1184 } |
(...skipping 12 matching lines...) Expand all Loading... |
1120 | 1197 |
1121 static inline ThreadCache* GetCacheIfPresent() { | 1198 static inline ThreadCache* GetCacheIfPresent() { |
1122 void* const p = ThreadCache::GetCacheIfPresent(); | 1199 void* const p = ThreadCache::GetCacheIfPresent(); |
1123 return reinterpret_cast<ThreadCache*>(p); | 1200 return reinterpret_cast<ThreadCache*>(p); |
1124 } | 1201 } |
1125 | 1202 |
1126 // This lets you call back to a given function pointer if ptr is invalid. | 1203 // This lets you call back to a given function pointer if ptr is invalid. |
1127 // It is used primarily by windows code which wants a specialized callback. | 1204 // It is used primarily by windows code which wants a specialized callback. |
1128 inline void do_free_with_callback(void* ptr, void (*invalid_free_fn)(void*)) { | 1205 inline void do_free_with_callback(void* ptr, void (*invalid_free_fn)(void*)) { |
1129 if (ptr == NULL) return; | 1206 if (ptr == NULL) return; |
1130 if (Static::pageheap() == NULL) { | 1207 ASSERT(Static::pageheap() != NULL); // Should not call free() before malloc() |
1131 // We called free() before malloc(). This can occur if the | |
1132 // (system) malloc() is called before tcmalloc is loaded, and then | |
1133 // free() is called after tcmalloc is loaded (and tc_free has | |
1134 // replaced free), but before the global constructor has run that | |
1135 // sets up the tcmalloc data structures. | |
1136 (*invalid_free_fn)(ptr); // Decide how to handle the bad free request | |
1137 return; | |
1138 } | |
1139 const PageID p = reinterpret_cast<uintptr_t>(ptr) >> kPageShift; | 1208 const PageID p = reinterpret_cast<uintptr_t>(ptr) >> kPageShift; |
1140 Span* span = NULL; | 1209 Span* span = NULL; |
1141 size_t cl = Static::pageheap()->GetSizeClassIfCached(p); | 1210 size_t cl = Static::pageheap()->GetSizeClassIfCached(p); |
1142 | 1211 |
1143 if (cl == 0) { | 1212 if (cl == 0) { |
1144 span = Static::pageheap()->GetDescriptor(p); | 1213 span = Static::pageheap()->GetDescriptor(p); |
1145 if (!span) { | 1214 if (!span) { |
1146 // span can be NULL because the pointer passed in is invalid | 1215 // span can be NULL because the pointer passed in is invalid |
1147 // (not something returned by malloc or friends), or because the | 1216 // (not something returned by malloc or friends), or because the |
1148 // pointer was allocated with some other allocator besides | 1217 // pointer was allocated with some other allocator besides |
(...skipping 32 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1181 } | 1250 } |
1182 Static::pageheap()->Delete(span); | 1251 Static::pageheap()->Delete(span); |
1183 } | 1252 } |
1184 } | 1253 } |
1185 | 1254 |
1186 // The default "do_free" that uses the default callback. | 1255 // The default "do_free" that uses the default callback. |
1187 inline void do_free(void* ptr) { | 1256 inline void do_free(void* ptr) { |
1188 return do_free_with_callback(ptr, &InvalidFree); | 1257 return do_free_with_callback(ptr, &InvalidFree); |
1189 } | 1258 } |
1190 | 1259 |
1191 // NOTE: some logic here is duplicated in GetOwnership (above), for | 1260 inline size_t GetSizeWithCallback(void* ptr, |
1192 // speed. If you change this function, look at that one too. | 1261 size_t (*invalid_getsize_fn)(void*)) { |
1193 inline size_t GetSizeWithCallback(const void* ptr, | |
1194 size_t (*invalid_getsize_fn)(const void*)) { | |
1195 if (ptr == NULL) | 1262 if (ptr == NULL) |
1196 return 0; | 1263 return 0; |
1197 const PageID p = reinterpret_cast<uintptr_t>(ptr) >> kPageShift; | 1264 const PageID p = reinterpret_cast<uintptr_t>(ptr) >> kPageShift; |
1198 size_t cl = Static::pageheap()->GetSizeClassIfCached(p); | 1265 size_t cl = Static::pageheap()->GetSizeClassIfCached(p); |
1199 if (cl != 0) { | 1266 if (cl != 0) { |
1200 return Static::sizemap()->ByteSizeForClass(cl); | 1267 return Static::sizemap()->ByteSizeForClass(cl); |
1201 } else { | 1268 } else { |
1202 const Span *span = Static::pageheap()->GetDescriptor(p); | 1269 Span *span = Static::pageheap()->GetDescriptor(p); |
1203 if (span == NULL) { // means we do not own this memory | 1270 if (span == NULL) { // means we do not own this memory |
1204 return (*invalid_getsize_fn)(ptr); | 1271 return (*invalid_getsize_fn)(ptr); |
1205 } else if (span->sizeclass != 0) { | 1272 } else if (span->sizeclass != 0) { |
1206 Static::pageheap()->CacheSizeClass(p, span->sizeclass); | 1273 Static::pageheap()->CacheSizeClass(p, span->sizeclass); |
1207 return Static::sizemap()->ByteSizeForClass(span->sizeclass); | 1274 return Static::sizemap()->ByteSizeForClass(span->sizeclass); |
1208 } else { | 1275 } else { |
1209 return span->length << kPageShift; | 1276 return span->length << kPageShift; |
1210 } | 1277 } |
1211 } | 1278 } |
1212 } | 1279 } |
1213 | 1280 |
1214 // This lets you call back to a given function pointer if ptr is invalid. | 1281 // This lets you call back to a given function pointer if ptr is invalid. |
1215 // It is used primarily by windows code which wants a specialized callback. | 1282 // It is used primarily by windows code which wants a specialized callback. |
1216 inline void* do_realloc_with_callback( | 1283 inline void* do_realloc_with_callback( |
1217 void* old_ptr, size_t new_size, | 1284 void* old_ptr, size_t new_size, |
1218 void (*invalid_free_fn)(void*), | 1285 void (*invalid_free_fn)(void*), |
1219 size_t (*invalid_get_size_fn)(const void*)) { | 1286 size_t (*invalid_get_size_fn)(void*)) { |
1220 AddRoomForMark(&new_size); | 1287 AddRoomForMark(&new_size); |
1221 // Get the size of the old entry | 1288 // Get the size of the old entry |
1222 const size_t old_size = GetSizeWithCallback(old_ptr, invalid_get_size_fn); | 1289 const size_t old_size = GetSizeWithCallback(old_ptr, invalid_get_size_fn); |
1223 | 1290 |
1224 // Reallocate if the new size is larger than the old size, | 1291 // Reallocate if the new size is larger than the old size, |
1225 // or if the new size is significantly smaller than the old size. | 1292 // or if the new size is significantly smaller than the old size. |
1226 // We do hysteresis to avoid resizing ping-pongs: | 1293 // We do hysteresis to avoid resizing ping-pongs: |
1227 // . If we need to grow, grow to max(new_size, old_size * 1.X) | 1294 // . If we need to grow, grow to max(new_size, old_size * 1.X) |
1228 // . Don't shrink unless new_size < old_size * 0.Y | 1295 // . Don't shrink unless new_size < old_size * 0.Y |
1229 // X and Y trade-off time for wasted space. For now we do 1.25 and 0.5. | 1296 // X and Y trade-off time for wasted space. For now we do 1.25 and 0.5. |
(...skipping 125 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1355 PrintStats(1); | 1422 PrintStats(1); |
1356 } | 1423 } |
1357 | 1424 |
1358 inline int do_mallopt(int cmd, int value) { | 1425 inline int do_mallopt(int cmd, int value) { |
1359 return 1; // Indicates error | 1426 return 1; // Indicates error |
1360 } | 1427 } |
1361 | 1428 |
1362 #ifdef HAVE_STRUCT_MALLINFO | 1429 #ifdef HAVE_STRUCT_MALLINFO |
1363 inline struct mallinfo do_mallinfo() { | 1430 inline struct mallinfo do_mallinfo() { |
1364 TCMallocStats stats; | 1431 TCMallocStats stats; |
1365 ExtractStats(&stats, NULL, NULL, NULL); | 1432 ExtractStats(&stats, NULL); |
1366 | 1433 |
1367 // Just some of the fields are filled in. | 1434 // Just some of the fields are filled in. |
1368 struct mallinfo info; | 1435 struct mallinfo info; |
1369 memset(&info, 0, sizeof(info)); | 1436 memset(&info, 0, sizeof(info)); |
1370 | 1437 |
1371 // Unfortunately, the struct contains "int" field, so some of the | 1438 // Unfortunately, the struct contains "int" field, so some of the |
1372 // size values will be truncated. | 1439 // size values will be truncated. |
1373 info.arena = static_cast<int>(stats.pageheap.system_bytes); | 1440 info.arena = static_cast<int>(stats.pageheap.system_bytes); |
1374 info.fsmblks = static_cast<int>(stats.thread_bytes | 1441 info.fsmblks = static_cast<int>(stats.thread_bytes |
1375 + stats.central_bytes | 1442 + stats.central_bytes |
(...skipping 103 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1479 } else { // allocation success | 1546 } else { // allocation success |
1480 return p; | 1547 return p; |
1481 } | 1548 } |
1482 #endif // PREANSINEW | 1549 #endif // PREANSINEW |
1483 } | 1550 } |
1484 } | 1551 } |
1485 | 1552 |
1486 } // end unnamed namespace | 1553 } // end unnamed namespace |
1487 | 1554 |
1488 // As promised, the definition of this function, declared above. | 1555 // As promised, the definition of this function, declared above. |
1489 size_t TCMallocImplementation::GetAllocatedSize(const void* ptr) { | 1556 size_t TCMallocImplementation::GetAllocatedSize(void* ptr) { |
1490 ASSERT(TCMallocImplementation::GetOwnership(ptr) | |
1491 != TCMallocImplementation::kNotOwned); | |
1492 return ExcludeSpaceForMark( | 1557 return ExcludeSpaceForMark( |
1493 GetSizeWithCallback(ptr, &InvalidGetAllocatedSize)); | 1558 GetSizeWithCallback(ptr, &InvalidGetAllocatedSize)); |
1494 } | 1559 } |
1495 | 1560 |
1496 void TCMallocImplementation::MarkThreadBusy() { | 1561 void TCMallocImplementation::MarkThreadBusy() { |
1497 // Allocate to force the creation of a thread cache, but avoid | 1562 // Allocate to force the creation of a thread cache, but avoid |
1498 // invoking any hooks. | 1563 // invoking any hooks. |
1499 do_free(do_malloc(0)); | 1564 do_free(do_malloc(0)); |
1500 } | 1565 } |
1501 | 1566 |
(...skipping 178 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1680 return do_mallopt(cmd, value); | 1745 return do_mallopt(cmd, value); |
1681 } | 1746 } |
1682 | 1747 |
1683 #ifdef HAVE_STRUCT_MALLINFO | 1748 #ifdef HAVE_STRUCT_MALLINFO |
1684 extern "C" PERFTOOLS_DLL_DECL struct mallinfo tc_mallinfo(void) __THROW { | 1749 extern "C" PERFTOOLS_DLL_DECL struct mallinfo tc_mallinfo(void) __THROW { |
1685 return do_mallinfo(); | 1750 return do_mallinfo(); |
1686 } | 1751 } |
1687 #endif | 1752 #endif |
1688 | 1753 |
1689 extern "C" PERFTOOLS_DLL_DECL size_t tc_malloc_size(void* ptr) __THROW { | 1754 extern "C" PERFTOOLS_DLL_DECL size_t tc_malloc_size(void* ptr) __THROW { |
1690 return MallocExtension::instance()->GetAllocatedSize(ptr); | 1755 return GetSizeWithCallback(ptr, &InvalidGetAllocatedSize); |
1691 } | 1756 } |
1692 | 1757 |
| 1758 |
| 1759 // Override __libc_memalign in libc on linux boxes specially. |
| 1760 // They have a bug in libc that causes them to (very rarely) allocate |
| 1761 // with __libc_memalign() yet deallocate with free() and the |
| 1762 // definitions above don't catch it. |
| 1763 // This function is an exception to the rule of calling MallocHook method |
| 1764 // from the stack frame of the allocation function; |
| 1765 // heap-checker handles this special case explicitly. |
| 1766 static void *MemalignOverride(size_t align, size_t size, const void *caller) |
| 1767 __THROW ATTRIBUTE_SECTION(google_malloc); |
| 1768 |
| 1769 static void *MemalignOverride(size_t align, size_t size, const void *caller) |
| 1770 __THROW { |
| 1771 void* result = do_memalign_or_cpp_memalign(align, size); |
| 1772 MallocHook::InvokeNewHook(result, size); |
| 1773 return result; |
| 1774 } |
| 1775 void *(*__MALLOC_HOOK_VOLATILE __memalign_hook)(size_t, size_t, const void *) =
MemalignOverride; |
1693 #endif // TCMALLOC_USING_DEBUGALLOCATION | 1776 #endif // TCMALLOC_USING_DEBUGALLOCATION |
1694 | 1777 |
1695 // ---Double free() debugging implementation ----------------------------------- | 1778 // ---Double free() debugging implementation ----------------------------------- |
1696 // We will put a mark at the extreme end of each allocation block. We make | 1779 // We will put a mark at the extreme end of each allocation block. We make |
1697 // sure that we always allocate enough "extra memory" that we can fit in the | 1780 // sure that we always allocate enough "extra memory" that we can fit in the |
1698 // mark, and still provide the requested usable region. If ever that mark is | 1781 // mark, and still provide the requested usable region. If ever that mark is |
1699 // not as expected, then we know that the user is corrupting memory beyond their | 1782 // not as expected, then we know that the user is corrupting memory beyond their |
1700 // request size, or that they have called free a second time without having | 1783 // request size, or that they have called free a second time without having |
1701 // the memory allocated (again). This allows us to spot most double free()s, | 1784 // the memory allocated (again). This allows us to spot most double free()s, |
1702 // but some can "slip by" or confuse our logic if the caller reallocates memory | 1785 // but some can "slip by" or confuse our logic if the caller reallocates memory |
(...skipping 30 matching lines...) Expand all Loading... |
1733 static void ValidateAllocatedRegion(void* ptr, size_t cl) {} | 1816 static void ValidateAllocatedRegion(void* ptr, size_t cl) {} |
1734 | 1817 |
1735 #else // TCMALLOC_VALIDATION | 1818 #else // TCMALLOC_VALIDATION |
1736 | 1819 |
1737 static void DieFromDoubleFree() { | 1820 static void DieFromDoubleFree() { |
1738 char* p = NULL; | 1821 char* p = NULL; |
1739 p++; | 1822 p++; |
1740 *p += 1; // Segv. | 1823 *p += 1; // Segv. |
1741 } | 1824 } |
1742 | 1825 |
1743 static size_t DieFromBadFreePointer(const void* unused) { | 1826 static size_t DieFromBadFreePointer(void* unused) { |
1744 char* p = NULL; | 1827 char* p = NULL; |
1745 p += 2; | 1828 p += 2; |
1746 *p += 2; // Segv. | 1829 *p += 2; // Segv. |
1747 return 0; | 1830 return 0; |
1748 } | 1831 } |
1749 | 1832 |
1750 static void DieFromMemoryCorruption() { | 1833 static void DieFromMemoryCorruption() { |
1751 char* p = NULL; | 1834 char* p = NULL; |
1752 p += 3; | 1835 p += 3; |
1753 *p += 3; // Segv. | 1836 *p += 3; // Segv. |
(...skipping 99 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1853 *mark = ~allocated_mark; // Distinctively not allocated. | 1936 *mark = ~allocated_mark; // Distinctively not allocated. |
1854 } | 1937 } |
1855 | 1938 |
1856 static void MarkAllocatedRegion(void* ptr) { | 1939 static void MarkAllocatedRegion(void* ptr) { |
1857 if (ptr == NULL) return; | 1940 if (ptr == NULL) return; |
1858 MarkType* mark = GetMarkLocation(ptr); | 1941 MarkType* mark = GetMarkLocation(ptr); |
1859 *mark = GetMarkValue(ptr, mark); | 1942 *mark = GetMarkValue(ptr, mark); |
1860 } | 1943 } |
1861 | 1944 |
1862 #endif // TCMALLOC_VALIDATION | 1945 #endif // TCMALLOC_VALIDATION |
OLD | NEW |