OLD | NEW |
1 // Copyright (c) 2005, Google Inc. | 1 // Copyright (c) 2005, Google Inc. |
2 // All rights reserved. | 2 // All rights reserved. |
3 // | 3 // |
4 // Redistribution and use in source and binary forms, with or without | 4 // Redistribution and use in source and binary forms, with or without |
5 // modification, are permitted provided that the following conditions are | 5 // modification, are permitted provided that the following conditions are |
6 // met: | 6 // met: |
7 // | 7 // |
8 // * Redistributions of source code must retain the above copyright | 8 // * Redistributions of source code must retain the above copyright |
9 // notice, this list of conditions and the following disclaimer. | 9 // notice, this list of conditions and the following disclaimer. |
10 // * Redistributions in binary form must reproduce the above | 10 // * Redistributions in binary form must reproduce the above |
(...skipping 69 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
80 // TODO: Bias reclamation to larger addresses | 80 // TODO: Bias reclamation to larger addresses |
81 // TODO: implement mallinfo/mallopt | 81 // TODO: implement mallinfo/mallopt |
82 // TODO: Better testing | 82 // TODO: Better testing |
83 // | 83 // |
84 // 9/28/2003 (new page-level allocator replaces ptmalloc2): | 84 // 9/28/2003 (new page-level allocator replaces ptmalloc2): |
85 // * malloc/free of small objects goes from ~300 ns to ~50 ns. | 85 // * malloc/free of small objects goes from ~300 ns to ~50 ns. |
86 // * allocation of a reasonably complicated struct | 86 // * allocation of a reasonably complicated struct |
87 // goes from about 1100 ns to about 300 ns. | 87 // goes from about 1100 ns to about 300 ns. |
88 | 88 |
89 #include "config.h" | 89 #include "config.h" |
90 #include <gperftools/tcmalloc.h> | 90 #include <google/tcmalloc.h> |
91 | 91 |
92 #include <errno.h> // for ENOMEM, EINVAL, errno | 92 #include <errno.h> // for ENOMEM, EINVAL, errno |
93 #ifdef HAVE_SYS_CDEFS_H | 93 #ifdef HAVE_SYS_CDEFS_H |
94 #include <sys/cdefs.h> // for __THROW | 94 #include <sys/cdefs.h> // for __THROW |
95 #endif | 95 #endif |
96 #if defined HAVE_STDINT_H | 96 #if defined HAVE_STDINT_H |
97 #include <stdint.h> | 97 #include <stdint.h> |
98 #elif defined HAVE_INTTYPES_H | 98 #elif defined HAVE_INTTYPES_H |
99 #include <inttypes.h> | 99 #include <inttypes.h> |
100 #else | 100 #else |
101 #include <sys/types.h> | 101 #include <sys/types.h> |
102 #endif | 102 #endif |
103 #include <stddef.h> // for size_t, NULL | 103 #include <stddef.h> // for size_t, NULL |
104 #include <stdlib.h> // for getenv | 104 #include <stdlib.h> // for getenv |
105 #include <string.h> // for strcmp, memset, strlen, etc | 105 #include <string.h> // for strcmp, memset, strlen, etc |
106 #ifdef HAVE_UNISTD_H | 106 #ifdef HAVE_UNISTD_H |
107 #include <unistd.h> // for getpagesize, write, etc | 107 #include <unistd.h> // for getpagesize, write, etc |
108 #endif | 108 #endif |
109 #include <algorithm> // for max, min | 109 #include <algorithm> // for max, min |
110 #include <limits> // for numeric_limits | 110 #include <limits> // for numeric_limits |
111 #include <new> // for nothrow_t (ptr only), etc | 111 #include <new> // for nothrow_t (ptr only), etc |
112 #include <vector> // for vector | 112 #include <vector> // for vector |
113 | 113 |
114 #include <gperftools/malloc_extension.h> | 114 #include <google/malloc_extension.h> |
115 #include <gperftools/malloc_hook.h> // for MallocHook | 115 #include <google/malloc_hook.h> // for MallocHook |
116 #include "base/basictypes.h" // for int64 | 116 #include "base/basictypes.h" // for int64 |
117 #include "base/commandlineflags.h" // for RegisterFlagValidator, etc | 117 #include "base/commandlineflags.h" // for RegisterFlagValidator, etc |
118 #include "base/dynamic_annotations.h" // for RunningOnValgrind | 118 #include "base/dynamic_annotations.h" // for RunningOnValgrind |
119 #include "base/spinlock.h" // for SpinLockHolder | 119 #include "base/spinlock.h" // for SpinLockHolder |
120 #include "central_freelist.h" // for CentralFreeListPadded | 120 #include "central_freelist.h" // for CentralFreeListPadded |
121 #include "common.h" // for StackTrace, kPageShift, etc | 121 #include "common.h" // for StackTrace, kPageShift, etc |
122 #include "internal_logging.h" // for ASSERT, TCMalloc_Printer, etc | 122 #include "internal_logging.h" // for ASSERT, TCMalloc_Printer, etc |
123 #include "linked_list.h" // for SLL_SetNext | 123 #include "linked_list.h" // for SLL_SetNext |
124 #include "malloc_hook-inl.h" // for MallocHook::InvokeNewHook, etc | 124 #include "malloc_hook-inl.h" // for MallocHook::InvokeNewHook, etc |
125 #include "page_heap.h" // for PageHeap, PageHeap::Stats | 125 #include "page_heap.h" // for PageHeap, PageHeap::Stats |
(...skipping 14 matching lines...) Expand all Loading... |
140 # include <sys/malloc.h> | 140 # include <sys/malloc.h> |
141 # elif defined(HAVE_MALLOC_MALLOC_H) | 141 # elif defined(HAVE_MALLOC_MALLOC_H) |
142 # include <malloc/malloc.h> | 142 # include <malloc/malloc.h> |
143 # endif | 143 # endif |
144 #endif | 144 #endif |
145 | 145 |
146 #if (defined(_WIN32) && !defined(__CYGWIN__) && !defined(__CYGWIN32__)) && !defi
ned(WIN32_OVERRIDE_ALLOCATORS) | 146 #if (defined(_WIN32) && !defined(__CYGWIN__) && !defined(__CYGWIN32__)) && !defi
ned(WIN32_OVERRIDE_ALLOCATORS) |
147 # define WIN32_DO_PATCHING 1 | 147 # define WIN32_DO_PATCHING 1 |
148 #endif | 148 #endif |
149 | 149 |
150 // Some windows file somewhere (at least on cygwin) #define's small (!) | |
151 #undef small | |
152 | |
153 using STL_NAMESPACE::max; | 150 using STL_NAMESPACE::max; |
154 using STL_NAMESPACE::numeric_limits; | 151 using STL_NAMESPACE::numeric_limits; |
155 using STL_NAMESPACE::vector; | 152 using STL_NAMESPACE::vector; |
156 | 153 |
157 #include "libc_override.h" | 154 #include "libc_override.h" |
158 | 155 |
159 // __THROW is defined in glibc (via <sys/cdefs.h>). It means, | 156 // __THROW is defined in glibc (via <sys/cdefs.h>). It means, |
160 // counter-intuitively, "This function will never throw an exception." | 157 // counter-intuitively, "This function will never throw an exception." |
161 // It's an optional optimization tool, but we may need to use it to | 158 // It's an optional optimization tool, but we may need to use it to |
162 // match glibc prototypes. | 159 // match glibc prototypes. |
163 #ifndef __THROW // I guess we're not on a glibc system | 160 #ifndef __THROW // I guess we're not on a glibc system |
164 # define __THROW // __THROW is just an optimization, so ok to make it "" | 161 # define __THROW // __THROW is just an optimization, so ok to make it "" |
165 #endif | 162 #endif |
166 | 163 |
167 using tcmalloc::AlignmentForSize; | 164 using tcmalloc::AlignmentForSize; |
168 using tcmalloc::kLog; | |
169 using tcmalloc::kCrash; | |
170 using tcmalloc::kCrashWithStats; | |
171 using tcmalloc::Log; | |
172 using tcmalloc::PageHeap; | 165 using tcmalloc::PageHeap; |
173 using tcmalloc::PageHeapAllocator; | 166 using tcmalloc::PageHeapAllocator; |
174 using tcmalloc::SizeMap; | 167 using tcmalloc::SizeMap; |
175 using tcmalloc::Span; | 168 using tcmalloc::Span; |
176 using tcmalloc::StackTrace; | 169 using tcmalloc::StackTrace; |
177 using tcmalloc::Static; | 170 using tcmalloc::Static; |
178 using tcmalloc::ThreadCache; | 171 using tcmalloc::ThreadCache; |
179 | 172 |
180 DECLARE_int64(tcmalloc_sample_parameter); | 173 DECLARE_int64(tcmalloc_sample_parameter); |
181 DECLARE_double(tcmalloc_release_rate); | 174 DECLARE_double(tcmalloc_release_rate); |
(...skipping 97 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
279 | 272 |
280 static int tc_new_mode = 0; // See tc_set_new_mode(). | 273 static int tc_new_mode = 0; // See tc_set_new_mode(). |
281 | 274 |
282 // Routines such as free() and realloc() catch some erroneous pointers | 275 // Routines such as free() and realloc() catch some erroneous pointers |
283 // passed to them, and invoke the below when they do. (An erroneous pointer | 276 // passed to them, and invoke the below when they do. (An erroneous pointer |
284 // won't be caught if it's within a valid span or a stale span for which | 277 // won't be caught if it's within a valid span or a stale span for which |
285 // the pagemap cache has a non-zero sizeclass.) This is a cheap (source-editing | 278 // the pagemap cache has a non-zero sizeclass.) This is a cheap (source-editing |
286 // required) kind of exception handling for these routines. | 279 // required) kind of exception handling for these routines. |
287 namespace { | 280 namespace { |
288 void InvalidFree(void* ptr) { | 281 void InvalidFree(void* ptr) { |
289 Log(kCrash, __FILE__, __LINE__, "Attempt to free invalid pointer", ptr); | 282 CRASH("Attempt to free invalid pointer: %p\n", ptr); |
290 } | 283 } |
291 | 284 |
292 size_t InvalidGetSizeForRealloc(const void* old_ptr) { | 285 size_t InvalidGetSizeForRealloc(void* old_ptr) { |
293 Log(kCrash, __FILE__, __LINE__, | 286 CRASH("Attempt to realloc invalid pointer: %p\n", old_ptr); |
294 "Attempt to realloc invalid pointer", old_ptr); | |
295 return 0; | 287 return 0; |
296 } | 288 } |
297 | 289 |
298 size_t InvalidGetAllocatedSize(const void* ptr) { | 290 size_t InvalidGetAllocatedSize(void* ptr) { |
299 Log(kCrash, __FILE__, __LINE__, | 291 CRASH("Attempt to get the size of an invalid pointer: %p\n", ptr); |
300 "Attempt to get the size of an invalid pointer", ptr); | |
301 return 0; | 292 return 0; |
302 } | 293 } |
303 } // unnamed namespace | 294 } // unnamed namespace |
304 | 295 |
305 // Extract interesting stats | 296 // Extract interesting stats |
306 struct TCMallocStats { | 297 struct TCMallocStats { |
307 uint64_t thread_bytes; // Bytes in thread caches | 298 uint64_t thread_bytes; // Bytes in thread caches |
308 uint64_t central_bytes; // Bytes in central cache | 299 uint64_t central_bytes; // Bytes in central cache |
309 uint64_t transfer_bytes; // Bytes in central transfer cache | 300 uint64_t transfer_bytes; // Bytes in central transfer cache |
310 uint64_t metadata_bytes; // Bytes alloced for metadata | 301 uint64_t metadata_bytes; // Bytes alloced for metadata |
311 PageHeap::Stats pageheap; // Stats from page heap | 302 PageHeap::Stats pageheap; // Stats from page heap |
312 }; | 303 }; |
313 | 304 |
314 // Get stats into "r". Also get per-size-class counts if class_count != NULL | 305 // Get stats into "r". Also get per-size-class counts if class_count != NULL |
315 static void ExtractStats(TCMallocStats* r, uint64_t* class_count, | 306 static void ExtractStats(TCMallocStats* r, uint64_t* class_count) { |
316 PageHeap::SmallSpanStats* small_spans, | |
317 PageHeap::LargeSpanStats* large_spans) { | |
318 r->central_bytes = 0; | 307 r->central_bytes = 0; |
319 r->transfer_bytes = 0; | 308 r->transfer_bytes = 0; |
320 for (int cl = 0; cl < kNumClasses; ++cl) { | 309 for (int cl = 0; cl < kNumClasses; ++cl) { |
321 const int length = Static::central_cache()[cl].length(); | 310 const int length = Static::central_cache()[cl].length(); |
322 const int tc_length = Static::central_cache()[cl].tc_length(); | 311 const int tc_length = Static::central_cache()[cl].tc_length(); |
323 const size_t cache_overhead = Static::central_cache()[cl].OverheadBytes(); | 312 const size_t cache_overhead = Static::central_cache()[cl].OverheadBytes(); |
324 const size_t size = static_cast<uint64_t>( | 313 const size_t size = static_cast<uint64_t>( |
325 Static::sizemap()->ByteSizeForClass(cl)); | 314 Static::sizemap()->ByteSizeForClass(cl)); |
326 r->central_bytes += (size * length) + cache_overhead; | 315 r->central_bytes += (size * length) + cache_overhead; |
327 r->transfer_bytes += (size * tc_length); | 316 r->transfer_bytes += (size * tc_length); |
328 if (class_count) class_count[cl] = length + tc_length; | 317 if (class_count) class_count[cl] = length + tc_length; |
329 } | 318 } |
330 | 319 |
331 // Add stats from per-thread heaps | 320 // Add stats from per-thread heaps |
332 r->thread_bytes = 0; | 321 r->thread_bytes = 0; |
333 { // scope | 322 { // scope |
334 SpinLockHolder h(Static::pageheap_lock()); | 323 SpinLockHolder h(Static::pageheap_lock()); |
335 ThreadCache::GetThreadStats(&r->thread_bytes, class_count); | 324 ThreadCache::GetThreadStats(&r->thread_bytes, class_count); |
336 r->metadata_bytes = tcmalloc::metadata_system_bytes(); | 325 r->metadata_bytes = tcmalloc::metadata_system_bytes(); |
337 r->pageheap = Static::pageheap()->stats(); | 326 r->pageheap = Static::pageheap()->stats(); |
338 if (small_spans != NULL) { | |
339 Static::pageheap()->GetSmallSpanStats(small_spans); | |
340 } | |
341 if (large_spans != NULL) { | |
342 Static::pageheap()->GetLargeSpanStats(large_spans); | |
343 } | |
344 } | 327 } |
345 } | 328 } |
346 | 329 |
347 static double PagesToMiB(uint64_t pages) { | |
348 return (pages << kPageShift) / 1048576.0; | |
349 } | |
350 | |
351 // WRITE stats to "out" | 330 // WRITE stats to "out" |
352 static void DumpStats(TCMalloc_Printer* out, int level) { | 331 static void DumpStats(TCMalloc_Printer* out, int level) { |
353 TCMallocStats stats; | 332 TCMallocStats stats; |
354 uint64_t class_count[kNumClasses]; | 333 uint64_t class_count[kNumClasses]; |
355 PageHeap::SmallSpanStats small; | 334 ExtractStats(&stats, (level >= 2 ? class_count : NULL)); |
356 PageHeap::LargeSpanStats large; | |
357 if (level >= 2) { | |
358 ExtractStats(&stats, class_count, &small, &large); | |
359 } else { | |
360 ExtractStats(&stats, NULL, NULL, NULL); | |
361 } | |
362 | 335 |
363 static const double MiB = 1048576.0; | 336 static const double MiB = 1048576.0; |
364 | 337 |
365 const uint64_t virtual_memory_used = (stats.pageheap.system_bytes | 338 const uint64_t virtual_memory_used = (stats.pageheap.system_bytes |
366 + stats.metadata_bytes); | 339 + stats.metadata_bytes); |
367 const uint64_t physical_memory_used = (virtual_memory_used | 340 const uint64_t physical_memory_used = (virtual_memory_used |
368 - stats.pageheap.unmapped_bytes); | 341 - stats.pageheap.unmapped_bytes); |
369 const uint64_t bytes_in_use_by_app = (physical_memory_used | 342 const uint64_t bytes_in_use_by_app = (physical_memory_used |
370 - stats.metadata_bytes | 343 - stats.metadata_bytes |
371 - stats.pageheap.free_bytes | 344 - stats.pageheap.free_bytes |
(...skipping 52 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
424 cumulative += class_bytes; | 397 cumulative += class_bytes; |
425 out->printf("class %3d [ %8" PRIuS " bytes ] : " | 398 out->printf("class %3d [ %8" PRIuS " bytes ] : " |
426 "%8" PRIu64 " objs; %5.1f MiB; %5.1f cum MiB\n", | 399 "%8" PRIu64 " objs; %5.1f MiB; %5.1f cum MiB\n", |
427 cl, Static::sizemap()->ByteSizeForClass(cl), | 400 cl, Static::sizemap()->ByteSizeForClass(cl), |
428 class_count[cl], | 401 class_count[cl], |
429 class_bytes / MiB, | 402 class_bytes / MiB, |
430 cumulative / MiB); | 403 cumulative / MiB); |
431 } | 404 } |
432 } | 405 } |
433 | 406 |
434 // append page heap info | 407 SpinLockHolder h(Static::pageheap_lock()); |
435 int nonempty_sizes = 0; | 408 Static::pageheap()->Dump(out); |
436 for (int s = 0; s < kMaxPages; s++) { | |
437 if (small.normal_length[s] + small.returned_length[s] > 0) { | |
438 nonempty_sizes++; | |
439 } | |
440 } | |
441 out->printf("------------------------------------------------\n"); | |
442 out->printf("PageHeap: %d sizes; %6.1f MiB free; %6.1f MiB unmapped\n", | |
443 nonempty_sizes, stats.pageheap.free_bytes / MiB, | |
444 stats.pageheap.unmapped_bytes / MiB); | |
445 out->printf("------------------------------------------------\n"); | |
446 uint64_t total_normal = 0; | |
447 uint64_t total_returned = 0; | |
448 for (int s = 0; s < kMaxPages; s++) { | |
449 const int n_length = small.normal_length[s]; | |
450 const int r_length = small.returned_length[s]; | |
451 if (n_length + r_length > 0) { | |
452 uint64_t n_pages = s * n_length; | |
453 uint64_t r_pages = s * r_length; | |
454 total_normal += n_pages; | |
455 total_returned += r_pages; | |
456 out->printf("%6u pages * %6u spans ~ %6.1f MiB; %6.1f MiB cum" | |
457 "; unmapped: %6.1f MiB; %6.1f MiB cum\n", | |
458 s, | |
459 (n_length + r_length), | |
460 PagesToMiB(n_pages + r_pages), | |
461 PagesToMiB(total_normal + total_returned), | |
462 PagesToMiB(r_pages), | |
463 PagesToMiB(total_returned)); | |
464 } | |
465 } | |
466 | |
467 total_normal += large.normal_pages; | |
468 total_returned += large.returned_pages; | |
469 out->printf(">255 large * %6u spans ~ %6.1f MiB; %6.1f MiB cum" | |
470 "; unmapped: %6.1f MiB; %6.1f MiB cum\n", | |
471 static_cast<unsigned int>(large.spans), | |
472 PagesToMiB(large.normal_pages + large.returned_pages), | |
473 PagesToMiB(total_normal + total_returned), | |
474 PagesToMiB(large.returned_pages), | |
475 PagesToMiB(total_returned)); | |
476 } | 409 } |
477 } | 410 } |
478 | 411 |
479 static void PrintStats(int level) { | 412 static void PrintStats(int level) { |
480 const int kBufferSize = 16 << 10; | 413 const int kBufferSize = 16 << 10; |
481 char* buffer = new char[kBufferSize]; | 414 char* buffer = new char[kBufferSize]; |
482 TCMalloc_Printer printer(buffer, kBufferSize); | 415 TCMalloc_Printer printer(buffer, kBufferSize); |
483 DumpStats(&printer, level); | 416 DumpStats(&printer, level); |
484 write(STDERR_FILENO, buffer, strlen(buffer)); | 417 write(STDERR_FILENO, buffer, strlen(buffer)); |
485 delete[] buffer; | 418 delete[] buffer; |
486 } | 419 } |
487 | 420 |
488 static void** DumpHeapGrowthStackTraces() { | 421 static void** DumpHeapGrowthStackTraces() { |
489 // Count how much space we need | 422 // Count how much space we need |
490 int needed_slots = 0; | 423 int needed_slots = 0; |
491 { | 424 { |
492 SpinLockHolder h(Static::pageheap_lock()); | 425 SpinLockHolder h(Static::pageheap_lock()); |
493 for (StackTrace* t = Static::growth_stacks(); | 426 for (StackTrace* t = Static::growth_stacks(); |
494 t != NULL; | 427 t != NULL; |
495 t = reinterpret_cast<StackTrace*>( | 428 t = reinterpret_cast<StackTrace*>( |
496 t->stack[tcmalloc::kMaxStackDepth-1])) { | 429 t->stack[tcmalloc::kMaxStackDepth-1])) { |
497 needed_slots += 3 + t->depth; | 430 needed_slots += 3 + t->depth; |
498 } | 431 } |
499 needed_slots += 100; // Slop in case list grows | 432 needed_slots += 100; // Slop in case list grows |
500 needed_slots += needed_slots/8; // An extra 12.5% slop | 433 needed_slots += needed_slots/8; // An extra 12.5% slop |
501 } | 434 } |
502 | 435 |
503 void** result = new void*[needed_slots]; | 436 void** result = new void*[needed_slots]; |
504 if (result == NULL) { | 437 if (result == NULL) { |
505 Log(kLog, __FILE__, __LINE__, | 438 MESSAGE("tcmalloc: allocation failed for stack trace slots", |
506 "tcmalloc: allocation failed for stack trace slots", | 439 needed_slots * sizeof(*result)); |
507 needed_slots * sizeof(*result)); | |
508 return NULL; | 440 return NULL; |
509 } | 441 } |
510 | 442 |
511 SpinLockHolder h(Static::pageheap_lock()); | 443 SpinLockHolder h(Static::pageheap_lock()); |
512 int used_slots = 0; | 444 int used_slots = 0; |
513 for (StackTrace* t = Static::growth_stacks(); | 445 for (StackTrace* t = Static::growth_stacks(); |
514 t != NULL; | 446 t != NULL; |
515 t = reinterpret_cast<StackTrace*>( | 447 t = reinterpret_cast<StackTrace*>( |
516 t->stack[tcmalloc::kMaxStackDepth-1])) { | 448 t->stack[tcmalloc::kMaxStackDepth-1])) { |
517 ASSERT(used_slots < needed_slots); // Need to leave room for terminator | 449 ASSERT(used_slots < needed_slots); // Need to leave room for terminator |
(...skipping 105 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
623 | 555 |
624 virtual void Ranges(void* arg, RangeFunction func) { | 556 virtual void Ranges(void* arg, RangeFunction func) { |
625 IterateOverRanges(arg, func); | 557 IterateOverRanges(arg, func); |
626 } | 558 } |
627 | 559 |
628 virtual bool GetNumericProperty(const char* name, size_t* value) { | 560 virtual bool GetNumericProperty(const char* name, size_t* value) { |
629 ASSERT(name != NULL); | 561 ASSERT(name != NULL); |
630 | 562 |
631 if (strcmp(name, "generic.current_allocated_bytes") == 0) { | 563 if (strcmp(name, "generic.current_allocated_bytes") == 0) { |
632 TCMallocStats stats; | 564 TCMallocStats stats; |
633 ExtractStats(&stats, NULL, NULL, NULL); | 565 ExtractStats(&stats, NULL); |
634 *value = stats.pageheap.system_bytes | 566 *value = stats.pageheap.system_bytes |
635 - stats.thread_bytes | 567 - stats.thread_bytes |
636 - stats.central_bytes | 568 - stats.central_bytes |
637 - stats.transfer_bytes | 569 - stats.transfer_bytes |
638 - stats.pageheap.free_bytes | 570 - stats.pageheap.free_bytes |
639 - stats.pageheap.unmapped_bytes; | 571 - stats.pageheap.unmapped_bytes; |
640 return true; | 572 return true; |
641 } | 573 } |
642 | 574 |
643 if (strcmp(name, "generic.heap_size") == 0) { | 575 if (strcmp(name, "generic.heap_size") == 0) { |
644 TCMallocStats stats; | 576 TCMallocStats stats; |
645 ExtractStats(&stats, NULL, NULL, NULL); | 577 ExtractStats(&stats, NULL); |
646 *value = stats.pageheap.system_bytes; | 578 *value = stats.pageheap.system_bytes; |
647 return true; | 579 return true; |
648 } | 580 } |
649 | 581 |
650 if (strcmp(name, "tcmalloc.slack_bytes") == 0) { | 582 if (strcmp(name, "tcmalloc.slack_bytes") == 0) { |
651 // Kept for backwards compatibility. Now defined externally as: | 583 // Kept for backwards compatibility. Now defined externally as: |
652 // pageheap_free_bytes + pageheap_unmapped_bytes. | 584 // pageheap_free_bytes + pageheap_unmapped_bytes. |
653 SpinLockHolder l(Static::pageheap_lock()); | 585 SpinLockHolder l(Static::pageheap_lock()); |
654 PageHeap::Stats stats = Static::pageheap()->stats(); | 586 PageHeap::Stats stats = Static::pageheap()->stats(); |
655 *value = stats.free_bytes + stats.unmapped_bytes; | 587 *value = stats.free_bytes + stats.unmapped_bytes; |
(...skipping 13 matching lines...) Expand all Loading... |
669 } | 601 } |
670 | 602 |
671 if (strcmp(name, "tcmalloc.max_total_thread_cache_bytes") == 0) { | 603 if (strcmp(name, "tcmalloc.max_total_thread_cache_bytes") == 0) { |
672 SpinLockHolder l(Static::pageheap_lock()); | 604 SpinLockHolder l(Static::pageheap_lock()); |
673 *value = ThreadCache::overall_thread_cache_size(); | 605 *value = ThreadCache::overall_thread_cache_size(); |
674 return true; | 606 return true; |
675 } | 607 } |
676 | 608 |
677 if (strcmp(name, "tcmalloc.current_total_thread_cache_bytes") == 0) { | 609 if (strcmp(name, "tcmalloc.current_total_thread_cache_bytes") == 0) { |
678 TCMallocStats stats; | 610 TCMallocStats stats; |
679 ExtractStats(&stats, NULL, NULL, NULL); | 611 ExtractStats(&stats, NULL); |
680 *value = stats.thread_bytes; | 612 *value = stats.thread_bytes; |
681 return true; | 613 return true; |
682 } | 614 } |
683 | 615 |
684 return false; | 616 return false; |
685 } | 617 } |
686 | 618 |
687 virtual bool SetNumericProperty(const char* name, size_t value) { | 619 virtual bool SetNumericProperty(const char* name, size_t value) { |
688 ASSERT(name != NULL); | 620 ASSERT(name != NULL); |
689 | 621 |
(...skipping 60 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
750 const size_t alloc_size = Static::sizemap()->ByteSizeForClass(cl); | 682 const size_t alloc_size = Static::sizemap()->ByteSizeForClass(cl); |
751 return alloc_size; | 683 return alloc_size; |
752 } else { | 684 } else { |
753 return tcmalloc::pages(size) << kPageShift; | 685 return tcmalloc::pages(size) << kPageShift; |
754 } | 686 } |
755 } | 687 } |
756 | 688 |
757 // This just calls GetSizeWithCallback, but because that's in an | 689 // This just calls GetSizeWithCallback, but because that's in an |
758 // unnamed namespace, we need to move the definition below it in the | 690 // unnamed namespace, we need to move the definition below it in the |
759 // file. | 691 // file. |
760 virtual size_t GetAllocatedSize(const void* ptr); | 692 virtual size_t GetAllocatedSize(void* ptr); |
761 | 693 |
762 // This duplicates some of the logic in GetSizeWithCallback, but is | 694 // This duplicates some of the logic in GetSizeWithCallback, but is |
763 // faster. This is important on OS X, where this function is called | 695 // faster. This is important on OS X, where this function is called |
764 // on every allocation operation. | 696 // on every allocation operation. |
765 virtual Ownership GetOwnership(const void* ptr) { | 697 virtual Ownership GetOwnership(const void* ptr) { |
766 const PageID p = reinterpret_cast<uintptr_t>(ptr) >> kPageShift; | 698 const PageID p = reinterpret_cast<uintptr_t>(ptr) >> kPageShift; |
767 // The rest of tcmalloc assumes that all allocated pointers use at | 699 // The rest of tcmalloc assumes that all allocated pointers use at |
768 // most kAddressBits bits. If ptr doesn't, then it definitely | 700 // most kAddressBits bits. If ptr doesn't, then it definitely |
769 // wasn't alloacted by tcmalloc. | 701 // wasn't alloacted by tcmalloc. |
770 if ((p >> (kAddressBits - kPageShift)) > 0) { | 702 if ((p >> (kAddressBits - kPageShift)) > 0) { |
(...skipping 53 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
824 MallocExtension::FreeListInfo i; | 756 MallocExtension::FreeListInfo i; |
825 i.min_object_size = prev_class_size + 1; | 757 i.min_object_size = prev_class_size + 1; |
826 i.max_object_size = Static::sizemap()->ByteSizeForClass(cl); | 758 i.max_object_size = Static::sizemap()->ByteSizeForClass(cl); |
827 i.total_bytes_free = | 759 i.total_bytes_free = |
828 class_count[cl] * Static::sizemap()->ByteSizeForClass(cl); | 760 class_count[cl] * Static::sizemap()->ByteSizeForClass(cl); |
829 i.type = kThreadCacheType; | 761 i.type = kThreadCacheType; |
830 v->push_back(i); | 762 v->push_back(i); |
831 } | 763 } |
832 | 764 |
833 // append page heap info | 765 // append page heap info |
834 PageHeap::SmallSpanStats small; | 766 int64 page_count_normal[kMaxPages]; |
835 PageHeap::LargeSpanStats large; | 767 int64 page_count_returned[kMaxPages]; |
| 768 int64 span_count_normal; |
| 769 int64 span_count_returned; |
836 { | 770 { |
837 SpinLockHolder h(Static::pageheap_lock()); | 771 SpinLockHolder h(Static::pageheap_lock()); |
838 Static::pageheap()->GetSmallSpanStats(&small); | 772 Static::pageheap()->GetClassSizes(page_count_normal, |
839 Static::pageheap()->GetLargeSpanStats(&large); | 773 page_count_returned, |
| 774 &span_count_normal, |
| 775 &span_count_returned); |
840 } | 776 } |
841 | 777 |
842 // large spans: mapped | 778 // spans: mapped |
843 MallocExtension::FreeListInfo span_info; | 779 MallocExtension::FreeListInfo span_info; |
844 span_info.type = kLargeSpanType; | 780 span_info.type = kLargeSpanType; |
845 span_info.max_object_size = (numeric_limits<size_t>::max)(); | 781 span_info.max_object_size = (numeric_limits<size_t>::max)(); |
846 span_info.min_object_size = kMaxPages << kPageShift; | 782 span_info.min_object_size = kMaxPages << kPageShift; |
847 span_info.total_bytes_free = large.normal_pages << kPageShift; | 783 span_info.total_bytes_free = span_count_normal << kPageShift; |
848 v->push_back(span_info); | 784 v->push_back(span_info); |
849 | 785 |
850 // large spans: unmapped | 786 // spans: unmapped |
851 span_info.type = kLargeUnmappedSpanType; | 787 span_info.type = kLargeUnmappedSpanType; |
852 span_info.total_bytes_free = large.returned_pages << kPageShift; | 788 span_info.total_bytes_free = span_count_returned << kPageShift; |
853 v->push_back(span_info); | 789 v->push_back(span_info); |
854 | 790 |
855 // small spans | |
856 for (int s = 1; s < kMaxPages; s++) { | 791 for (int s = 1; s < kMaxPages; s++) { |
857 MallocExtension::FreeListInfo i; | 792 MallocExtension::FreeListInfo i; |
858 i.max_object_size = (s << kPageShift); | 793 i.max_object_size = (s << kPageShift); |
859 i.min_object_size = ((s - 1) << kPageShift); | 794 i.min_object_size = ((s - 1) << kPageShift); |
860 | 795 |
861 i.type = kPageHeapType; | 796 i.type = kPageHeapType; |
862 i.total_bytes_free = (s << kPageShift) * small.normal_length[s]; | 797 i.total_bytes_free = (s << kPageShift) * page_count_normal[s]; |
863 v->push_back(i); | 798 v->push_back(i); |
864 | 799 |
865 i.type = kPageHeapUnmappedType; | 800 i.type = kPageHeapUnmappedType; |
866 i.total_bytes_free = (s << kPageShift) * small.returned_length[s]; | 801 i.total_bytes_free = (s << kPageShift) * page_count_returned[s]; |
867 v->push_back(i); | 802 v->push_back(i); |
868 } | 803 } |
869 } | 804 } |
870 }; | 805 }; |
871 | 806 |
872 // The constructor allocates an object to ensure that initialization | 807 // The constructor allocates an object to ensure that initialization |
873 // runs before main(), and therefore we do not have a chance to become | 808 // runs before main(), and therefore we do not have a chance to become |
874 // multi-threaded before initialization. We also create the TSD key | 809 // multi-threaded before initialization. We also create the TSD key |
875 // here. Presumably by the time this constructor runs, glibc is in | 810 // here. Presumably by the time this constructor runs, glibc is in |
876 // good enough shape to handle pthread_key_create(). | 811 // good enough shape to handle pthread_key_create(). |
877 // | 812 // |
878 // The constructor also takes the opportunity to tell STL to use | 813 // The constructor also takes the opportunity to tell STL to use |
879 // tcmalloc. We want to do this early, before construct time, so | 814 // tcmalloc. We want to do this early, before construct time, so |
880 // all user STL allocations go through tcmalloc (which works really | 815 // all user STL allocations go through tcmalloc (which works really |
881 // well for STL). | 816 // well for STL). |
882 // | 817 // |
883 // The destructor prints stats when the program exits. | 818 // The destructor prints stats when the program exits. |
884 static int tcmallocguard_refcount = 0; // no lock needed: runs before main() | 819 static int tcmallocguard_refcount = 0; // no lock needed: runs before main() |
885 TCMallocGuard::TCMallocGuard() { | 820 TCMallocGuard::TCMallocGuard() { |
886 if (tcmallocguard_refcount++ == 0) { | 821 if (tcmallocguard_refcount++ == 0) { |
887 #ifdef HAVE_TLS // this is true if the cc/ld/libc combo support TLS | 822 #ifdef HAVE_TLS // this is true if the cc/ld/libc combo support TLS |
888 // Check whether the kernel also supports TLS (needs to happen at runtime) | 823 // Check whether the kernel also supports TLS (needs to happen at runtime) |
889 tcmalloc::CheckIfKernelSupportsTLS(); | 824 tcmalloc::CheckIfKernelSupportsTLS(); |
890 #endif | 825 #endif |
891 ReplaceSystemAlloc(); // defined in libc_override_*.h | 826 ReplaceSystemAlloc(); // defined in libc_override_*.h |
| 827 #if defined(__APPLE__) |
| 828 // To break the recursive call of malloc, as malloc -> TCMALLOC_MESSAGE |
| 829 // -> snprintf -> localeconv_l -> malloc, on MacOS. |
| 830 char buf[32]; |
| 831 snprintf(buf, sizeof(buf), "%d", tcmallocguard_refcount); |
| 832 #endif |
892 tc_free(tc_malloc(1)); | 833 tc_free(tc_malloc(1)); |
893 ThreadCache::InitTSD(); | 834 ThreadCache::InitTSD(); |
894 tc_free(tc_malloc(1)); | 835 tc_free(tc_malloc(1)); |
895 // Either we, or debugallocation.cc, or valgrind will control memory | 836 // Either we, or debugallocation.cc, or valgrind will control memory |
896 // management. We register our extension if we're the winner. | 837 // management. We register our extension if we're the winner. |
897 #ifdef TCMALLOC_USING_DEBUGALLOCATION | 838 #ifdef TCMALLOC_USING_DEBUGALLOCATION |
898 // Let debugallocation register its extension. | 839 // Let debugallocation register its extension. |
899 #else | 840 #else |
900 if (RunningOnValgrind()) { | 841 if (RunningOnValgrind()) { |
901 // Let Valgrind uses its own malloc (so don't register our extension). | 842 // Let Valgrind uses its own malloc (so don't register our extension). |
(...skipping 186 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1088 | 1029 |
1089 static inline ThreadCache* GetCacheIfPresent() { | 1030 static inline ThreadCache* GetCacheIfPresent() { |
1090 void* const p = ThreadCache::GetCacheIfPresent(); | 1031 void* const p = ThreadCache::GetCacheIfPresent(); |
1091 return reinterpret_cast<ThreadCache*>(p); | 1032 return reinterpret_cast<ThreadCache*>(p); |
1092 } | 1033 } |
1093 | 1034 |
1094 // This lets you call back to a given function pointer if ptr is invalid. | 1035 // This lets you call back to a given function pointer if ptr is invalid. |
1095 // It is used primarily by windows code which wants a specialized callback. | 1036 // It is used primarily by windows code which wants a specialized callback. |
1096 inline void do_free_with_callback(void* ptr, void (*invalid_free_fn)(void*)) { | 1037 inline void do_free_with_callback(void* ptr, void (*invalid_free_fn)(void*)) { |
1097 if (ptr == NULL) return; | 1038 if (ptr == NULL) return; |
1098 if (Static::pageheap() == NULL) { | 1039 ASSERT(Static::pageheap() != NULL); // Should not call free() before malloc() |
1099 // We called free() before malloc(). This can occur if the | |
1100 // (system) malloc() is called before tcmalloc is loaded, and then | |
1101 // free() is called after tcmalloc is loaded (and tc_free has | |
1102 // replaced free), but before the global constructor has run that | |
1103 // sets up the tcmalloc data structures. | |
1104 (*invalid_free_fn)(ptr); // Decide how to handle the bad free request | |
1105 return; | |
1106 } | |
1107 const PageID p = reinterpret_cast<uintptr_t>(ptr) >> kPageShift; | 1040 const PageID p = reinterpret_cast<uintptr_t>(ptr) >> kPageShift; |
1108 Span* span = NULL; | 1041 Span* span = NULL; |
1109 size_t cl = Static::pageheap()->GetSizeClassIfCached(p); | 1042 size_t cl = Static::pageheap()->GetSizeClassIfCached(p); |
1110 | 1043 |
1111 if (cl == 0) { | 1044 if (cl == 0) { |
1112 span = Static::pageheap()->GetDescriptor(p); | 1045 span = Static::pageheap()->GetDescriptor(p); |
1113 if (!span) { | 1046 if (!span) { |
1114 // span can be NULL because the pointer passed in is invalid | 1047 // span can be NULL because the pointer passed in is invalid |
1115 // (not something returned by malloc or friends), or because the | 1048 // (not something returned by malloc or friends), or because the |
1116 // pointer was allocated with some other allocator besides | 1049 // pointer was allocated with some other allocator besides |
(...skipping 31 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1148 } | 1081 } |
1149 } | 1082 } |
1150 | 1083 |
1151 // The default "do_free" that uses the default callback. | 1084 // The default "do_free" that uses the default callback. |
1152 inline void do_free(void* ptr) { | 1085 inline void do_free(void* ptr) { |
1153 return do_free_with_callback(ptr, &InvalidFree); | 1086 return do_free_with_callback(ptr, &InvalidFree); |
1154 } | 1087 } |
1155 | 1088 |
1156 // NOTE: some logic here is duplicated in GetOwnership (above), for | 1089 // NOTE: some logic here is duplicated in GetOwnership (above), for |
1157 // speed. If you change this function, look at that one too. | 1090 // speed. If you change this function, look at that one too. |
1158 inline size_t GetSizeWithCallback(const void* ptr, | 1091 inline size_t GetSizeWithCallback(void* ptr, |
1159 size_t (*invalid_getsize_fn)(const void*)) { | 1092 size_t (*invalid_getsize_fn)(void*)) { |
1160 if (ptr == NULL) | 1093 if (ptr == NULL) |
1161 return 0; | 1094 return 0; |
1162 const PageID p = reinterpret_cast<uintptr_t>(ptr) >> kPageShift; | 1095 const PageID p = reinterpret_cast<uintptr_t>(ptr) >> kPageShift; |
1163 size_t cl = Static::pageheap()->GetSizeClassIfCached(p); | 1096 size_t cl = Static::pageheap()->GetSizeClassIfCached(p); |
1164 if (cl != 0) { | 1097 if (cl != 0) { |
1165 return Static::sizemap()->ByteSizeForClass(cl); | 1098 return Static::sizemap()->ByteSizeForClass(cl); |
1166 } else { | 1099 } else { |
1167 const Span *span = Static::pageheap()->GetDescriptor(p); | 1100 const Span *span = Static::pageheap()->GetDescriptor(p); |
1168 if (span == NULL) { // means we do not own this memory | 1101 if (span == NULL) { // means we do not own this memory |
1169 return (*invalid_getsize_fn)(ptr); | 1102 return (*invalid_getsize_fn)(ptr); |
1170 } else if (span->sizeclass != 0) { | 1103 } else if (span->sizeclass != 0) { |
1171 Static::pageheap()->CacheSizeClass(p, span->sizeclass); | 1104 Static::pageheap()->CacheSizeClass(p, span->sizeclass); |
1172 return Static::sizemap()->ByteSizeForClass(span->sizeclass); | 1105 return Static::sizemap()->ByteSizeForClass(span->sizeclass); |
1173 } else { | 1106 } else { |
1174 return span->length << kPageShift; | 1107 return span->length << kPageShift; |
1175 } | 1108 } |
1176 } | 1109 } |
1177 } | 1110 } |
1178 | 1111 |
1179 // This lets you call back to a given function pointer if ptr is invalid. | 1112 // This lets you call back to a given function pointer if ptr is invalid. |
1180 // It is used primarily by windows code which wants a specialized callback. | 1113 // It is used primarily by windows code which wants a specialized callback. |
1181 inline void* do_realloc_with_callback( | 1114 inline void* do_realloc_with_callback( |
1182 void* old_ptr, size_t new_size, | 1115 void* old_ptr, size_t new_size, |
1183 void (*invalid_free_fn)(void*), | 1116 void (*invalid_free_fn)(void*), |
1184 size_t (*invalid_get_size_fn)(const void*)) { | 1117 size_t (*invalid_get_size_fn)(void*)) { |
1185 // Get the size of the old entry | 1118 // Get the size of the old entry |
1186 const size_t old_size = GetSizeWithCallback(old_ptr, invalid_get_size_fn); | 1119 const size_t old_size = GetSizeWithCallback(old_ptr, invalid_get_size_fn); |
1187 | 1120 |
1188 // Reallocate if the new size is larger than the old size, | 1121 // Reallocate if the new size is larger than the old size, |
1189 // or if the new size is significantly smaller than the old size. | 1122 // or if the new size is significantly smaller than the old size. |
1190 // We do hysteresis to avoid resizing ping-pongs: | 1123 // We do hysteresis to avoid resizing ping-pongs: |
1191 // . If we need to grow, grow to max(new_size, old_size * 1.X) | 1124 // . If we need to grow, grow to max(new_size, old_size * 1.X) |
1192 // . Don't shrink unless new_size < old_size * 0.Y | 1125 // . Don't shrink unless new_size < old_size * 0.Y |
1193 // X and Y trade-off time for wasted space. For now we do 1.25 and 0.5. | 1126 // X and Y trade-off time for wasted space. For now we do 1.25 and 0.5. |
1194 const int lower_bound_to_grow = old_size + old_size / 4; | 1127 const int lower_bound_to_grow = old_size + old_size / 4; |
(...skipping 120 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1315 PrintStats(1); | 1248 PrintStats(1); |
1316 } | 1249 } |
1317 | 1250 |
1318 inline int do_mallopt(int cmd, int value) { | 1251 inline int do_mallopt(int cmd, int value) { |
1319 return 1; // Indicates error | 1252 return 1; // Indicates error |
1320 } | 1253 } |
1321 | 1254 |
1322 #ifdef HAVE_STRUCT_MALLINFO | 1255 #ifdef HAVE_STRUCT_MALLINFO |
1323 inline struct mallinfo do_mallinfo() { | 1256 inline struct mallinfo do_mallinfo() { |
1324 TCMallocStats stats; | 1257 TCMallocStats stats; |
1325 ExtractStats(&stats, NULL, NULL, NULL); | 1258 ExtractStats(&stats, NULL); |
1326 | 1259 |
1327 // Just some of the fields are filled in. | 1260 // Just some of the fields are filled in. |
1328 struct mallinfo info; | 1261 struct mallinfo info; |
1329 memset(&info, 0, sizeof(info)); | 1262 memset(&info, 0, sizeof(info)); |
1330 | 1263 |
1331 // Unfortunately, the struct contains "int" field, so some of the | 1264 // Unfortunately, the struct contains "int" field, so some of the |
1332 // size values will be truncated. | 1265 // size values will be truncated. |
1333 info.arena = static_cast<int>(stats.pageheap.system_bytes); | 1266 info.arena = static_cast<int>(stats.pageheap.system_bytes); |
1334 info.fsmblks = static_cast<int>(stats.thread_bytes | 1267 info.fsmblks = static_cast<int>(stats.thread_bytes |
1335 + stats.central_bytes | 1268 + stats.central_bytes |
(...skipping 103 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1439 } else { // allocation success | 1372 } else { // allocation success |
1440 return p; | 1373 return p; |
1441 } | 1374 } |
1442 #endif // PREANSINEW | 1375 #endif // PREANSINEW |
1443 } | 1376 } |
1444 } | 1377 } |
1445 | 1378 |
1446 } // end unnamed namespace | 1379 } // end unnamed namespace |
1447 | 1380 |
1448 // As promised, the definition of this function, declared above. | 1381 // As promised, the definition of this function, declared above. |
1449 size_t TCMallocImplementation::GetAllocatedSize(const void* ptr) { | 1382 size_t TCMallocImplementation::GetAllocatedSize(void* ptr) { |
1450 ASSERT(TCMallocImplementation::GetOwnership(ptr) | 1383 ASSERT(TCMallocImplementation::GetOwnership(ptr) |
1451 != TCMallocImplementation::kNotOwned); | 1384 != TCMallocImplementation::kNotOwned); |
1452 return GetSizeWithCallback(ptr, &InvalidGetAllocatedSize); | 1385 return GetSizeWithCallback(ptr, &InvalidGetAllocatedSize); |
1453 } | 1386 } |
1454 | 1387 |
1455 void TCMallocImplementation::MarkThreadBusy() { | 1388 void TCMallocImplementation::MarkThreadBusy() { |
1456 // Allocate to force the creation of a thread cache, but avoid | 1389 // Allocate to force the creation of a thread cache, but avoid |
1457 // invoking any hooks. | 1390 // invoking any hooks. |
1458 do_free(do_malloc(0)); | 1391 do_free(do_malloc(0)); |
1459 } | 1392 } |
(...skipping 183 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1643 extern "C" PERFTOOLS_DLL_DECL struct mallinfo tc_mallinfo(void) __THROW { | 1576 extern "C" PERFTOOLS_DLL_DECL struct mallinfo tc_mallinfo(void) __THROW { |
1644 return do_mallinfo(); | 1577 return do_mallinfo(); |
1645 } | 1578 } |
1646 #endif | 1579 #endif |
1647 | 1580 |
1648 extern "C" PERFTOOLS_DLL_DECL size_t tc_malloc_size(void* ptr) __THROW { | 1581 extern "C" PERFTOOLS_DLL_DECL size_t tc_malloc_size(void* ptr) __THROW { |
1649 return MallocExtension::instance()->GetAllocatedSize(ptr); | 1582 return MallocExtension::instance()->GetAllocatedSize(ptr); |
1650 } | 1583 } |
1651 | 1584 |
1652 #endif // TCMALLOC_USING_DEBUGALLOCATION | 1585 #endif // TCMALLOC_USING_DEBUGALLOCATION |
OLD | NEW |