Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(39)

Side by Side Diff: third_party/tcmalloc/vendor/src/tcmalloc.cc

Issue 9702045: Update the tcmalloc vendor branch to r144 (gperftools 2.0), and README.chromium. (Closed) Base URL: svn://svn.chromium.org/chrome/trunk/src
Patch Set: Created 8 years, 9 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch | Annotate | Revision Log
OLDNEW
1 // Copyright (c) 2005, Google Inc. 1 // Copyright (c) 2005, Google Inc.
2 // All rights reserved. 2 // All rights reserved.
3 // 3 //
4 // Redistribution and use in source and binary forms, with or without 4 // Redistribution and use in source and binary forms, with or without
5 // modification, are permitted provided that the following conditions are 5 // modification, are permitted provided that the following conditions are
6 // met: 6 // met:
7 // 7 //
8 // * Redistributions of source code must retain the above copyright 8 // * Redistributions of source code must retain the above copyright
9 // notice, this list of conditions and the following disclaimer. 9 // notice, this list of conditions and the following disclaimer.
10 // * Redistributions in binary form must reproduce the above 10 // * Redistributions in binary form must reproduce the above
(...skipping 69 matching lines...) Expand 10 before | Expand all | Expand 10 after
80 // TODO: Bias reclamation to larger addresses 80 // TODO: Bias reclamation to larger addresses
81 // TODO: implement mallinfo/mallopt 81 // TODO: implement mallinfo/mallopt
82 // TODO: Better testing 82 // TODO: Better testing
83 // 83 //
84 // 9/28/2003 (new page-level allocator replaces ptmalloc2): 84 // 9/28/2003 (new page-level allocator replaces ptmalloc2):
85 // * malloc/free of small objects goes from ~300 ns to ~50 ns. 85 // * malloc/free of small objects goes from ~300 ns to ~50 ns.
86 // * allocation of a reasonably complicated struct 86 // * allocation of a reasonably complicated struct
87 // goes from about 1100 ns to about 300 ns. 87 // goes from about 1100 ns to about 300 ns.
88 88
89 #include "config.h" 89 #include "config.h"
90 #include <google/tcmalloc.h> 90 #include <gperftools/tcmalloc.h>
91 91
92 #include <errno.h> // for ENOMEM, EINVAL, errno 92 #include <errno.h> // for ENOMEM, EINVAL, errno
93 #ifdef HAVE_SYS_CDEFS_H 93 #ifdef HAVE_SYS_CDEFS_H
94 #include <sys/cdefs.h> // for __THROW 94 #include <sys/cdefs.h> // for __THROW
95 #endif 95 #endif
96 #if defined HAVE_STDINT_H 96 #if defined HAVE_STDINT_H
97 #include <stdint.h> 97 #include <stdint.h>
98 #elif defined HAVE_INTTYPES_H 98 #elif defined HAVE_INTTYPES_H
99 #include <inttypes.h> 99 #include <inttypes.h>
100 #else 100 #else
101 #include <sys/types.h> 101 #include <sys/types.h>
102 #endif 102 #endif
103 #include <stddef.h> // for size_t, NULL 103 #include <stddef.h> // for size_t, NULL
104 #include <stdlib.h> // for getenv 104 #include <stdlib.h> // for getenv
105 #include <string.h> // for strcmp, memset, strlen, etc 105 #include <string.h> // for strcmp, memset, strlen, etc
106 #ifdef HAVE_UNISTD_H 106 #ifdef HAVE_UNISTD_H
107 #include <unistd.h> // for getpagesize, write, etc 107 #include <unistd.h> // for getpagesize, write, etc
108 #endif 108 #endif
109 #include <algorithm> // for max, min 109 #include <algorithm> // for max, min
110 #include <limits> // for numeric_limits 110 #include <limits> // for numeric_limits
111 #include <new> // for nothrow_t (ptr only), etc 111 #include <new> // for nothrow_t (ptr only), etc
112 #include <vector> // for vector 112 #include <vector> // for vector
113 113
114 #include <google/malloc_extension.h> 114 #include <gperftools/malloc_extension.h>
115 #include <google/malloc_hook.h> // for MallocHook 115 #include <gperftools/malloc_hook.h> // for MallocHook
116 #include "base/basictypes.h" // for int64 116 #include "base/basictypes.h" // for int64
117 #include "base/commandlineflags.h" // for RegisterFlagValidator, etc 117 #include "base/commandlineflags.h" // for RegisterFlagValidator, etc
118 #include "base/dynamic_annotations.h" // for RunningOnValgrind 118 #include "base/dynamic_annotations.h" // for RunningOnValgrind
119 #include "base/spinlock.h" // for SpinLockHolder 119 #include "base/spinlock.h" // for SpinLockHolder
120 #include "central_freelist.h" // for CentralFreeListPadded 120 #include "central_freelist.h" // for CentralFreeListPadded
121 #include "common.h" // for StackTrace, kPageShift, etc 121 #include "common.h" // for StackTrace, kPageShift, etc
122 #include "internal_logging.h" // for ASSERT, TCMalloc_Printer, etc 122 #include "internal_logging.h" // for ASSERT, TCMalloc_Printer, etc
123 #include "linked_list.h" // for SLL_SetNext 123 #include "linked_list.h" // for SLL_SetNext
124 #include "malloc_hook-inl.h" // for MallocHook::InvokeNewHook, etc 124 #include "malloc_hook-inl.h" // for MallocHook::InvokeNewHook, etc
125 #include "page_heap.h" // for PageHeap, PageHeap::Stats 125 #include "page_heap.h" // for PageHeap, PageHeap::Stats
(...skipping 14 matching lines...) Expand all
140 # include <sys/malloc.h> 140 # include <sys/malloc.h>
141 # elif defined(HAVE_MALLOC_MALLOC_H) 141 # elif defined(HAVE_MALLOC_MALLOC_H)
142 # include <malloc/malloc.h> 142 # include <malloc/malloc.h>
143 # endif 143 # endif
144 #endif 144 #endif
145 145
146 #if (defined(_WIN32) && !defined(__CYGWIN__) && !defined(__CYGWIN32__)) && !defi ned(WIN32_OVERRIDE_ALLOCATORS) 146 #if (defined(_WIN32) && !defined(__CYGWIN__) && !defined(__CYGWIN32__)) && !defi ned(WIN32_OVERRIDE_ALLOCATORS)
147 # define WIN32_DO_PATCHING 1 147 # define WIN32_DO_PATCHING 1
148 #endif 148 #endif
149 149
150 // Some windows file somewhere (at least on cygwin) #define's small (!)
151 #undef small
152
150 using STL_NAMESPACE::max; 153 using STL_NAMESPACE::max;
151 using STL_NAMESPACE::numeric_limits; 154 using STL_NAMESPACE::numeric_limits;
152 using STL_NAMESPACE::vector; 155 using STL_NAMESPACE::vector;
153 156
154 #include "libc_override.h" 157 #include "libc_override.h"
155 158
156 // __THROW is defined in glibc (via <sys/cdefs.h>). It means, 159 // __THROW is defined in glibc (via <sys/cdefs.h>). It means,
157 // counter-intuitively, "This function will never throw an exception." 160 // counter-intuitively, "This function will never throw an exception."
158 // It's an optional optimization tool, but we may need to use it to 161 // It's an optional optimization tool, but we may need to use it to
159 // match glibc prototypes. 162 // match glibc prototypes.
160 #ifndef __THROW // I guess we're not on a glibc system 163 #ifndef __THROW // I guess we're not on a glibc system
161 # define __THROW // __THROW is just an optimization, so ok to make it "" 164 # define __THROW // __THROW is just an optimization, so ok to make it ""
162 #endif 165 #endif
163 166
164 using tcmalloc::AlignmentForSize; 167 using tcmalloc::AlignmentForSize;
168 using tcmalloc::kLog;
169 using tcmalloc::kCrash;
170 using tcmalloc::kCrashWithStats;
171 using tcmalloc::Log;
165 using tcmalloc::PageHeap; 172 using tcmalloc::PageHeap;
166 using tcmalloc::PageHeapAllocator; 173 using tcmalloc::PageHeapAllocator;
167 using tcmalloc::SizeMap; 174 using tcmalloc::SizeMap;
168 using tcmalloc::Span; 175 using tcmalloc::Span;
169 using tcmalloc::StackTrace; 176 using tcmalloc::StackTrace;
170 using tcmalloc::Static; 177 using tcmalloc::Static;
171 using tcmalloc::ThreadCache; 178 using tcmalloc::ThreadCache;
172 179
173 DECLARE_int64(tcmalloc_sample_parameter); 180 DECLARE_int64(tcmalloc_sample_parameter);
174 DECLARE_double(tcmalloc_release_rate); 181 DECLARE_double(tcmalloc_release_rate);
(...skipping 97 matching lines...) Expand 10 before | Expand all | Expand 10 after
272 279
273 static int tc_new_mode = 0; // See tc_set_new_mode(). 280 static int tc_new_mode = 0; // See tc_set_new_mode().
274 281
275 // Routines such as free() and realloc() catch some erroneous pointers 282 // Routines such as free() and realloc() catch some erroneous pointers
276 // passed to them, and invoke the below when they do. (An erroneous pointer 283 // passed to them, and invoke the below when they do. (An erroneous pointer
277 // won't be caught if it's within a valid span or a stale span for which 284 // won't be caught if it's within a valid span or a stale span for which
278 // the pagemap cache has a non-zero sizeclass.) This is a cheap (source-editing 285 // the pagemap cache has a non-zero sizeclass.) This is a cheap (source-editing
279 // required) kind of exception handling for these routines. 286 // required) kind of exception handling for these routines.
280 namespace { 287 namespace {
281 void InvalidFree(void* ptr) { 288 void InvalidFree(void* ptr) {
282 CRASH("Attempt to free invalid pointer: %p\n", ptr); 289 Log(kCrash, __FILE__, __LINE__, "Attempt to free invalid pointer", ptr);
283 } 290 }
284 291
285 size_t InvalidGetSizeForRealloc(void* old_ptr) { 292 size_t InvalidGetSizeForRealloc(const void* old_ptr) {
286 CRASH("Attempt to realloc invalid pointer: %p\n", old_ptr); 293 Log(kCrash, __FILE__, __LINE__,
294 "Attempt to realloc invalid pointer", old_ptr);
287 return 0; 295 return 0;
288 } 296 }
289 297
290 size_t InvalidGetAllocatedSize(void* ptr) { 298 size_t InvalidGetAllocatedSize(const void* ptr) {
291 CRASH("Attempt to get the size of an invalid pointer: %p\n", ptr); 299 Log(kCrash, __FILE__, __LINE__,
300 "Attempt to get the size of an invalid pointer", ptr);
292 return 0; 301 return 0;
293 } 302 }
294 } // unnamed namespace 303 } // unnamed namespace
295 304
296 // Extract interesting stats 305 // Extract interesting stats
297 struct TCMallocStats { 306 struct TCMallocStats {
298 uint64_t thread_bytes; // Bytes in thread caches 307 uint64_t thread_bytes; // Bytes in thread caches
299 uint64_t central_bytes; // Bytes in central cache 308 uint64_t central_bytes; // Bytes in central cache
300 uint64_t transfer_bytes; // Bytes in central transfer cache 309 uint64_t transfer_bytes; // Bytes in central transfer cache
301 uint64_t metadata_bytes; // Bytes alloced for metadata 310 uint64_t metadata_bytes; // Bytes alloced for metadata
302 PageHeap::Stats pageheap; // Stats from page heap 311 PageHeap::Stats pageheap; // Stats from page heap
303 }; 312 };
304 313
305 // Get stats into "r". Also get per-size-class counts if class_count != NULL 314 // Get stats into "r". Also get per-size-class counts if class_count != NULL
306 static void ExtractStats(TCMallocStats* r, uint64_t* class_count) { 315 static void ExtractStats(TCMallocStats* r, uint64_t* class_count,
316 PageHeap::SmallSpanStats* small_spans,
317 PageHeap::LargeSpanStats* large_spans) {
307 r->central_bytes = 0; 318 r->central_bytes = 0;
308 r->transfer_bytes = 0; 319 r->transfer_bytes = 0;
309 for (int cl = 0; cl < kNumClasses; ++cl) { 320 for (int cl = 0; cl < kNumClasses; ++cl) {
310 const int length = Static::central_cache()[cl].length(); 321 const int length = Static::central_cache()[cl].length();
311 const int tc_length = Static::central_cache()[cl].tc_length(); 322 const int tc_length = Static::central_cache()[cl].tc_length();
312 const size_t cache_overhead = Static::central_cache()[cl].OverheadBytes(); 323 const size_t cache_overhead = Static::central_cache()[cl].OverheadBytes();
313 const size_t size = static_cast<uint64_t>( 324 const size_t size = static_cast<uint64_t>(
314 Static::sizemap()->ByteSizeForClass(cl)); 325 Static::sizemap()->ByteSizeForClass(cl));
315 r->central_bytes += (size * length) + cache_overhead; 326 r->central_bytes += (size * length) + cache_overhead;
316 r->transfer_bytes += (size * tc_length); 327 r->transfer_bytes += (size * tc_length);
317 if (class_count) class_count[cl] = length + tc_length; 328 if (class_count) class_count[cl] = length + tc_length;
318 } 329 }
319 330
320 // Add stats from per-thread heaps 331 // Add stats from per-thread heaps
321 r->thread_bytes = 0; 332 r->thread_bytes = 0;
322 { // scope 333 { // scope
323 SpinLockHolder h(Static::pageheap_lock()); 334 SpinLockHolder h(Static::pageheap_lock());
324 ThreadCache::GetThreadStats(&r->thread_bytes, class_count); 335 ThreadCache::GetThreadStats(&r->thread_bytes, class_count);
325 r->metadata_bytes = tcmalloc::metadata_system_bytes(); 336 r->metadata_bytes = tcmalloc::metadata_system_bytes();
326 r->pageheap = Static::pageheap()->stats(); 337 r->pageheap = Static::pageheap()->stats();
338 if (small_spans != NULL) {
339 Static::pageheap()->GetSmallSpanStats(small_spans);
340 }
341 if (large_spans != NULL) {
342 Static::pageheap()->GetLargeSpanStats(large_spans);
343 }
327 } 344 }
328 } 345 }
329 346
347 static double PagesToMiB(uint64_t pages) {
348 return (pages << kPageShift) / 1048576.0;
349 }
350
330 // WRITE stats to "out" 351 // WRITE stats to "out"
331 static void DumpStats(TCMalloc_Printer* out, int level) { 352 static void DumpStats(TCMalloc_Printer* out, int level) {
332 TCMallocStats stats; 353 TCMallocStats stats;
333 uint64_t class_count[kNumClasses]; 354 uint64_t class_count[kNumClasses];
334 ExtractStats(&stats, (level >= 2 ? class_count : NULL)); 355 PageHeap::SmallSpanStats small;
356 PageHeap::LargeSpanStats large;
357 if (level >= 2) {
358 ExtractStats(&stats, class_count, &small, &large);
359 } else {
360 ExtractStats(&stats, NULL, NULL, NULL);
361 }
335 362
336 static const double MiB = 1048576.0; 363 static const double MiB = 1048576.0;
337 364
338 const uint64_t virtual_memory_used = (stats.pageheap.system_bytes 365 const uint64_t virtual_memory_used = (stats.pageheap.system_bytes
339 + stats.metadata_bytes); 366 + stats.metadata_bytes);
340 const uint64_t physical_memory_used = (virtual_memory_used 367 const uint64_t physical_memory_used = (virtual_memory_used
341 - stats.pageheap.unmapped_bytes); 368 - stats.pageheap.unmapped_bytes);
342 const uint64_t bytes_in_use_by_app = (physical_memory_used 369 const uint64_t bytes_in_use_by_app = (physical_memory_used
343 - stats.metadata_bytes 370 - stats.metadata_bytes
344 - stats.pageheap.free_bytes 371 - stats.pageheap.free_bytes
(...skipping 52 matching lines...) Expand 10 before | Expand all | Expand 10 after
397 cumulative += class_bytes; 424 cumulative += class_bytes;
398 out->printf("class %3d [ %8" PRIuS " bytes ] : " 425 out->printf("class %3d [ %8" PRIuS " bytes ] : "
399 "%8" PRIu64 " objs; %5.1f MiB; %5.1f cum MiB\n", 426 "%8" PRIu64 " objs; %5.1f MiB; %5.1f cum MiB\n",
400 cl, Static::sizemap()->ByteSizeForClass(cl), 427 cl, Static::sizemap()->ByteSizeForClass(cl),
401 class_count[cl], 428 class_count[cl],
402 class_bytes / MiB, 429 class_bytes / MiB,
403 cumulative / MiB); 430 cumulative / MiB);
404 } 431 }
405 } 432 }
406 433
407 SpinLockHolder h(Static::pageheap_lock()); 434 // append page heap info
408 Static::pageheap()->Dump(out); 435 int nonempty_sizes = 0;
436 for (int s = 0; s < kMaxPages; s++) {
437 if (small.normal_length[s] + small.returned_length[s] > 0) {
438 nonempty_sizes++;
439 }
440 }
441 out->printf("------------------------------------------------\n");
442 out->printf("PageHeap: %d sizes; %6.1f MiB free; %6.1f MiB unmapped\n",
443 nonempty_sizes, stats.pageheap.free_bytes / MiB,
444 stats.pageheap.unmapped_bytes / MiB);
445 out->printf("------------------------------------------------\n");
446 uint64_t total_normal = 0;
447 uint64_t total_returned = 0;
448 for (int s = 0; s < kMaxPages; s++) {
449 const int n_length = small.normal_length[s];
450 const int r_length = small.returned_length[s];
451 if (n_length + r_length > 0) {
452 uint64_t n_pages = s * n_length;
453 uint64_t r_pages = s * r_length;
454 total_normal += n_pages;
455 total_returned += r_pages;
456 out->printf("%6u pages * %6u spans ~ %6.1f MiB; %6.1f MiB cum"
457 "; unmapped: %6.1f MiB; %6.1f MiB cum\n",
458 s,
459 (n_length + r_length),
460 PagesToMiB(n_pages + r_pages),
461 PagesToMiB(total_normal + total_returned),
462 PagesToMiB(r_pages),
463 PagesToMiB(total_returned));
464 }
465 }
466
467 total_normal += large.normal_pages;
468 total_returned += large.returned_pages;
469 out->printf(">255 large * %6u spans ~ %6.1f MiB; %6.1f MiB cum"
470 "; unmapped: %6.1f MiB; %6.1f MiB cum\n",
471 static_cast<unsigned int>(large.spans),
472 PagesToMiB(large.normal_pages + large.returned_pages),
473 PagesToMiB(total_normal + total_returned),
474 PagesToMiB(large.returned_pages),
475 PagesToMiB(total_returned));
409 } 476 }
410 } 477 }
411 478
412 static void PrintStats(int level) { 479 static void PrintStats(int level) {
413 const int kBufferSize = 16 << 10; 480 const int kBufferSize = 16 << 10;
414 char* buffer = new char[kBufferSize]; 481 char* buffer = new char[kBufferSize];
415 TCMalloc_Printer printer(buffer, kBufferSize); 482 TCMalloc_Printer printer(buffer, kBufferSize);
416 DumpStats(&printer, level); 483 DumpStats(&printer, level);
417 write(STDERR_FILENO, buffer, strlen(buffer)); 484 write(STDERR_FILENO, buffer, strlen(buffer));
418 delete[] buffer; 485 delete[] buffer;
419 } 486 }
420 487
421 static void** DumpHeapGrowthStackTraces() { 488 static void** DumpHeapGrowthStackTraces() {
422 // Count how much space we need 489 // Count how much space we need
423 int needed_slots = 0; 490 int needed_slots = 0;
424 { 491 {
425 SpinLockHolder h(Static::pageheap_lock()); 492 SpinLockHolder h(Static::pageheap_lock());
426 for (StackTrace* t = Static::growth_stacks(); 493 for (StackTrace* t = Static::growth_stacks();
427 t != NULL; 494 t != NULL;
428 t = reinterpret_cast<StackTrace*>( 495 t = reinterpret_cast<StackTrace*>(
429 t->stack[tcmalloc::kMaxStackDepth-1])) { 496 t->stack[tcmalloc::kMaxStackDepth-1])) {
430 needed_slots += 3 + t->depth; 497 needed_slots += 3 + t->depth;
431 } 498 }
432 needed_slots += 100; // Slop in case list grows 499 needed_slots += 100; // Slop in case list grows
433 needed_slots += needed_slots/8; // An extra 12.5% slop 500 needed_slots += needed_slots/8; // An extra 12.5% slop
434 } 501 }
435 502
436 void** result = new void*[needed_slots]; 503 void** result = new void*[needed_slots];
437 if (result == NULL) { 504 if (result == NULL) {
438 MESSAGE("tcmalloc: allocation failed for stack trace slots", 505 Log(kLog, __FILE__, __LINE__,
439 needed_slots * sizeof(*result)); 506 "tcmalloc: allocation failed for stack trace slots",
507 needed_slots * sizeof(*result));
440 return NULL; 508 return NULL;
441 } 509 }
442 510
443 SpinLockHolder h(Static::pageheap_lock()); 511 SpinLockHolder h(Static::pageheap_lock());
444 int used_slots = 0; 512 int used_slots = 0;
445 for (StackTrace* t = Static::growth_stacks(); 513 for (StackTrace* t = Static::growth_stacks();
446 t != NULL; 514 t != NULL;
447 t = reinterpret_cast<StackTrace*>( 515 t = reinterpret_cast<StackTrace*>(
448 t->stack[tcmalloc::kMaxStackDepth-1])) { 516 t->stack[tcmalloc::kMaxStackDepth-1])) {
449 ASSERT(used_slots < needed_slots); // Need to leave room for terminator 517 ASSERT(used_slots < needed_slots); // Need to leave room for terminator
(...skipping 105 matching lines...) Expand 10 before | Expand all | Expand 10 after
555 623
556 virtual void Ranges(void* arg, RangeFunction func) { 624 virtual void Ranges(void* arg, RangeFunction func) {
557 IterateOverRanges(arg, func); 625 IterateOverRanges(arg, func);
558 } 626 }
559 627
560 virtual bool GetNumericProperty(const char* name, size_t* value) { 628 virtual bool GetNumericProperty(const char* name, size_t* value) {
561 ASSERT(name != NULL); 629 ASSERT(name != NULL);
562 630
563 if (strcmp(name, "generic.current_allocated_bytes") == 0) { 631 if (strcmp(name, "generic.current_allocated_bytes") == 0) {
564 TCMallocStats stats; 632 TCMallocStats stats;
565 ExtractStats(&stats, NULL); 633 ExtractStats(&stats, NULL, NULL, NULL);
566 *value = stats.pageheap.system_bytes 634 *value = stats.pageheap.system_bytes
567 - stats.thread_bytes 635 - stats.thread_bytes
568 - stats.central_bytes 636 - stats.central_bytes
569 - stats.transfer_bytes 637 - stats.transfer_bytes
570 - stats.pageheap.free_bytes 638 - stats.pageheap.free_bytes
571 - stats.pageheap.unmapped_bytes; 639 - stats.pageheap.unmapped_bytes;
572 return true; 640 return true;
573 } 641 }
574 642
575 if (strcmp(name, "generic.heap_size") == 0) { 643 if (strcmp(name, "generic.heap_size") == 0) {
576 TCMallocStats stats; 644 TCMallocStats stats;
577 ExtractStats(&stats, NULL); 645 ExtractStats(&stats, NULL, NULL, NULL);
578 *value = stats.pageheap.system_bytes; 646 *value = stats.pageheap.system_bytes;
579 return true; 647 return true;
580 } 648 }
581 649
582 if (strcmp(name, "tcmalloc.slack_bytes") == 0) { 650 if (strcmp(name, "tcmalloc.slack_bytes") == 0) {
583 // Kept for backwards compatibility. Now defined externally as: 651 // Kept for backwards compatibility. Now defined externally as:
584 // pageheap_free_bytes + pageheap_unmapped_bytes. 652 // pageheap_free_bytes + pageheap_unmapped_bytes.
585 SpinLockHolder l(Static::pageheap_lock()); 653 SpinLockHolder l(Static::pageheap_lock());
586 PageHeap::Stats stats = Static::pageheap()->stats(); 654 PageHeap::Stats stats = Static::pageheap()->stats();
587 *value = stats.free_bytes + stats.unmapped_bytes; 655 *value = stats.free_bytes + stats.unmapped_bytes;
588 return true; 656 return true;
589 } 657 }
590 658
659 if (strcmp(name, "tcmalloc.central_cache_free_bytes") == 0) {
660 TCMallocStats stats;
661 ExtractStats(&stats, NULL, NULL, NULL);
662 *value = stats.central_bytes;
663 return true;
664 }
665
666 if (strcmp(name, "tcmalloc.transfer_cache_free_bytes") == 0) {
667 TCMallocStats stats;
668 ExtractStats(&stats, NULL, NULL, NULL);
669 *value = stats.transfer_bytes;
670 return true;
671 }
672
673 if (strcmp(name, "tcmalloc.thread_cache_free_bytes") == 0) {
674 TCMallocStats stats;
675 ExtractStats(&stats, NULL, NULL, NULL);
676 *value = stats.thread_bytes;
677 return true;
678 }
679
591 if (strcmp(name, "tcmalloc.pageheap_free_bytes") == 0) { 680 if (strcmp(name, "tcmalloc.pageheap_free_bytes") == 0) {
592 SpinLockHolder l(Static::pageheap_lock()); 681 SpinLockHolder l(Static::pageheap_lock());
593 *value = Static::pageheap()->stats().free_bytes; 682 *value = Static::pageheap()->stats().free_bytes;
594 return true; 683 return true;
595 } 684 }
596 685
597 if (strcmp(name, "tcmalloc.pageheap_unmapped_bytes") == 0) { 686 if (strcmp(name, "tcmalloc.pageheap_unmapped_bytes") == 0) {
598 SpinLockHolder l(Static::pageheap_lock()); 687 SpinLockHolder l(Static::pageheap_lock());
599 *value = Static::pageheap()->stats().unmapped_bytes; 688 *value = Static::pageheap()->stats().unmapped_bytes;
600 return true; 689 return true;
601 } 690 }
602 691
603 if (strcmp(name, "tcmalloc.max_total_thread_cache_bytes") == 0) { 692 if (strcmp(name, "tcmalloc.max_total_thread_cache_bytes") == 0) {
604 SpinLockHolder l(Static::pageheap_lock()); 693 SpinLockHolder l(Static::pageheap_lock());
605 *value = ThreadCache::overall_thread_cache_size(); 694 *value = ThreadCache::overall_thread_cache_size();
606 return true; 695 return true;
607 } 696 }
608 697
609 if (strcmp(name, "tcmalloc.current_total_thread_cache_bytes") == 0) { 698 if (strcmp(name, "tcmalloc.current_total_thread_cache_bytes") == 0) {
610 TCMallocStats stats; 699 TCMallocStats stats;
611 ExtractStats(&stats, NULL); 700 ExtractStats(&stats, NULL, NULL, NULL);
612 *value = stats.thread_bytes; 701 *value = stats.thread_bytes;
613 return true; 702 return true;
614 } 703 }
615 704
616 return false; 705 return false;
617 } 706 }
618 707
619 virtual bool SetNumericProperty(const char* name, size_t value) { 708 virtual bool SetNumericProperty(const char* name, size_t value) {
620 ASSERT(name != NULL); 709 ASSERT(name != NULL);
621 710
(...skipping 60 matching lines...) Expand 10 before | Expand all | Expand 10 after
682 const size_t alloc_size = Static::sizemap()->ByteSizeForClass(cl); 771 const size_t alloc_size = Static::sizemap()->ByteSizeForClass(cl);
683 return alloc_size; 772 return alloc_size;
684 } else { 773 } else {
685 return tcmalloc::pages(size) << kPageShift; 774 return tcmalloc::pages(size) << kPageShift;
686 } 775 }
687 } 776 }
688 777
689 // This just calls GetSizeWithCallback, but because that's in an 778 // This just calls GetSizeWithCallback, but because that's in an
690 // unnamed namespace, we need to move the definition below it in the 779 // unnamed namespace, we need to move the definition below it in the
691 // file. 780 // file.
692 virtual size_t GetAllocatedSize(void* ptr); 781 virtual size_t GetAllocatedSize(const void* ptr);
693 782
694 // This duplicates some of the logic in GetSizeWithCallback, but is 783 // This duplicates some of the logic in GetSizeWithCallback, but is
695 // faster. This is important on OS X, where this function is called 784 // faster. This is important on OS X, where this function is called
696 // on every allocation operation. 785 // on every allocation operation.
697 virtual Ownership GetOwnership(const void* ptr) { 786 virtual Ownership GetOwnership(const void* ptr) {
698 const PageID p = reinterpret_cast<uintptr_t>(ptr) >> kPageShift; 787 const PageID p = reinterpret_cast<uintptr_t>(ptr) >> kPageShift;
699 // The rest of tcmalloc assumes that all allocated pointers use at 788 // The rest of tcmalloc assumes that all allocated pointers use at
700 // most kAddressBits bits. If ptr doesn't, then it definitely 789 // most kAddressBits bits. If ptr doesn't, then it definitely
701 // wasn't alloacted by tcmalloc. 790 // wasn't alloacted by tcmalloc.
702 if ((p >> (kAddressBits - kPageShift)) > 0) { 791 if ((p >> (kAddressBits - kPageShift)) > 0) {
(...skipping 53 matching lines...) Expand 10 before | Expand all | Expand 10 after
756 MallocExtension::FreeListInfo i; 845 MallocExtension::FreeListInfo i;
757 i.min_object_size = prev_class_size + 1; 846 i.min_object_size = prev_class_size + 1;
758 i.max_object_size = Static::sizemap()->ByteSizeForClass(cl); 847 i.max_object_size = Static::sizemap()->ByteSizeForClass(cl);
759 i.total_bytes_free = 848 i.total_bytes_free =
760 class_count[cl] * Static::sizemap()->ByteSizeForClass(cl); 849 class_count[cl] * Static::sizemap()->ByteSizeForClass(cl);
761 i.type = kThreadCacheType; 850 i.type = kThreadCacheType;
762 v->push_back(i); 851 v->push_back(i);
763 } 852 }
764 853
765 // append page heap info 854 // append page heap info
766 int64 page_count_normal[kMaxPages]; 855 PageHeap::SmallSpanStats small;
767 int64 page_count_returned[kMaxPages]; 856 PageHeap::LargeSpanStats large;
768 int64 span_count_normal;
769 int64 span_count_returned;
770 { 857 {
771 SpinLockHolder h(Static::pageheap_lock()); 858 SpinLockHolder h(Static::pageheap_lock());
772 Static::pageheap()->GetClassSizes(page_count_normal, 859 Static::pageheap()->GetSmallSpanStats(&small);
773 page_count_returned, 860 Static::pageheap()->GetLargeSpanStats(&large);
774 &span_count_normal,
775 &span_count_returned);
776 } 861 }
777 862
778 // spans: mapped 863 // large spans: mapped
779 MallocExtension::FreeListInfo span_info; 864 MallocExtension::FreeListInfo span_info;
780 span_info.type = kLargeSpanType; 865 span_info.type = kLargeSpanType;
781 span_info.max_object_size = (numeric_limits<size_t>::max)(); 866 span_info.max_object_size = (numeric_limits<size_t>::max)();
782 span_info.min_object_size = kMaxPages << kPageShift; 867 span_info.min_object_size = kMaxPages << kPageShift;
783 span_info.total_bytes_free = span_count_normal << kPageShift; 868 span_info.total_bytes_free = large.normal_pages << kPageShift;
784 v->push_back(span_info); 869 v->push_back(span_info);
785 870
786 // spans: unmapped 871 // large spans: unmapped
787 span_info.type = kLargeUnmappedSpanType; 872 span_info.type = kLargeUnmappedSpanType;
788 span_info.total_bytes_free = span_count_returned << kPageShift; 873 span_info.total_bytes_free = large.returned_pages << kPageShift;
789 v->push_back(span_info); 874 v->push_back(span_info);
790 875
876 // small spans
791 for (int s = 1; s < kMaxPages; s++) { 877 for (int s = 1; s < kMaxPages; s++) {
792 MallocExtension::FreeListInfo i; 878 MallocExtension::FreeListInfo i;
793 i.max_object_size = (s << kPageShift); 879 i.max_object_size = (s << kPageShift);
794 i.min_object_size = ((s - 1) << kPageShift); 880 i.min_object_size = ((s - 1) << kPageShift);
795 881
796 i.type = kPageHeapType; 882 i.type = kPageHeapType;
797 i.total_bytes_free = (s << kPageShift) * page_count_normal[s]; 883 i.total_bytes_free = (s << kPageShift) * small.normal_length[s];
798 v->push_back(i); 884 v->push_back(i);
799 885
800 i.type = kPageHeapUnmappedType; 886 i.type = kPageHeapUnmappedType;
801 i.total_bytes_free = (s << kPageShift) * page_count_returned[s]; 887 i.total_bytes_free = (s << kPageShift) * small.returned_length[s];
802 v->push_back(i); 888 v->push_back(i);
803 } 889 }
804 } 890 }
805 }; 891 };
806 892
807 // The constructor allocates an object to ensure that initialization 893 // The constructor allocates an object to ensure that initialization
808 // runs before main(), and therefore we do not have a chance to become 894 // runs before main(), and therefore we do not have a chance to become
809 // multi-threaded before initialization. We also create the TSD key 895 // multi-threaded before initialization. We also create the TSD key
810 // here. Presumably by the time this constructor runs, glibc is in 896 // here. Presumably by the time this constructor runs, glibc is in
811 // good enough shape to handle pthread_key_create(). 897 // good enough shape to handle pthread_key_create().
812 // 898 //
813 // The constructor also takes the opportunity to tell STL to use 899 // The constructor also takes the opportunity to tell STL to use
814 // tcmalloc. We want to do this early, before construct time, so 900 // tcmalloc. We want to do this early, before construct time, so
815 // all user STL allocations go through tcmalloc (which works really 901 // all user STL allocations go through tcmalloc (which works really
816 // well for STL). 902 // well for STL).
817 // 903 //
818 // The destructor prints stats when the program exits. 904 // The destructor prints stats when the program exits.
819 static int tcmallocguard_refcount = 0; // no lock needed: runs before main() 905 static int tcmallocguard_refcount = 0; // no lock needed: runs before main()
820 TCMallocGuard::TCMallocGuard() { 906 TCMallocGuard::TCMallocGuard() {
821 if (tcmallocguard_refcount++ == 0) { 907 if (tcmallocguard_refcount++ == 0) {
822 #ifdef HAVE_TLS // this is true if the cc/ld/libc combo support TLS 908 #ifdef HAVE_TLS // this is true if the cc/ld/libc combo support TLS
823 // Check whether the kernel also supports TLS (needs to happen at runtime) 909 // Check whether the kernel also supports TLS (needs to happen at runtime)
824 tcmalloc::CheckIfKernelSupportsTLS(); 910 tcmalloc::CheckIfKernelSupportsTLS();
825 #endif 911 #endif
826 ReplaceSystemAlloc(); // defined in libc_override_*.h 912 ReplaceSystemAlloc(); // defined in libc_override_*.h
827 #if defined(__APPLE__)
828 // To break the recursive call of malloc, as malloc -> TCMALLOC_MESSAGE
829 // -> snprintf -> localeconv_l -> malloc, on MacOS.
830 char buf[32];
831 snprintf(buf, sizeof(buf), "%d", tcmallocguard_refcount);
832 #endif
833 tc_free(tc_malloc(1)); 913 tc_free(tc_malloc(1));
834 ThreadCache::InitTSD(); 914 ThreadCache::InitTSD();
835 tc_free(tc_malloc(1)); 915 tc_free(tc_malloc(1));
836 // Either we, or debugallocation.cc, or valgrind will control memory 916 // Either we, or debugallocation.cc, or valgrind will control memory
837 // management. We register our extension if we're the winner. 917 // management. We register our extension if we're the winner.
838 #ifdef TCMALLOC_USING_DEBUGALLOCATION 918 #ifdef TCMALLOC_USING_DEBUGALLOCATION
839 // Let debugallocation register its extension. 919 // Let debugallocation register its extension.
840 #else 920 #else
841 if (RunningOnValgrind()) { 921 if (RunningOnValgrind()) {
842 // Let Valgrind uses its own malloc (so don't register our extension). 922 // Let Valgrind uses its own malloc (so don't register our extension).
(...skipping 186 matching lines...) Expand 10 before | Expand all | Expand 10 after
1029 1109
1030 static inline ThreadCache* GetCacheIfPresent() { 1110 static inline ThreadCache* GetCacheIfPresent() {
1031 void* const p = ThreadCache::GetCacheIfPresent(); 1111 void* const p = ThreadCache::GetCacheIfPresent();
1032 return reinterpret_cast<ThreadCache*>(p); 1112 return reinterpret_cast<ThreadCache*>(p);
1033 } 1113 }
1034 1114
1035 // This lets you call back to a given function pointer if ptr is invalid. 1115 // This lets you call back to a given function pointer if ptr is invalid.
1036 // It is used primarily by windows code which wants a specialized callback. 1116 // It is used primarily by windows code which wants a specialized callback.
1037 inline void do_free_with_callback(void* ptr, void (*invalid_free_fn)(void*)) { 1117 inline void do_free_with_callback(void* ptr, void (*invalid_free_fn)(void*)) {
1038 if (ptr == NULL) return; 1118 if (ptr == NULL) return;
1039 ASSERT(Static::pageheap() != NULL); // Should not call free() before malloc() 1119 if (Static::pageheap() == NULL) {
1120 // We called free() before malloc(). This can occur if the
1121 // (system) malloc() is called before tcmalloc is loaded, and then
1122 // free() is called after tcmalloc is loaded (and tc_free has
1123 // replaced free), but before the global constructor has run that
1124 // sets up the tcmalloc data structures.
1125 (*invalid_free_fn)(ptr); // Decide how to handle the bad free request
1126 return;
1127 }
1040 const PageID p = reinterpret_cast<uintptr_t>(ptr) >> kPageShift; 1128 const PageID p = reinterpret_cast<uintptr_t>(ptr) >> kPageShift;
1041 Span* span = NULL; 1129 Span* span = NULL;
1042 size_t cl = Static::pageheap()->GetSizeClassIfCached(p); 1130 size_t cl = Static::pageheap()->GetSizeClassIfCached(p);
1043 1131
1044 if (cl == 0) { 1132 if (cl == 0) {
1045 span = Static::pageheap()->GetDescriptor(p); 1133 span = Static::pageheap()->GetDescriptor(p);
1046 if (!span) { 1134 if (!span) {
1047 // span can be NULL because the pointer passed in is invalid 1135 // span can be NULL because the pointer passed in is invalid
1048 // (not something returned by malloc or friends), or because the 1136 // (not something returned by malloc or friends), or because the
1049 // pointer was allocated with some other allocator besides 1137 // pointer was allocated with some other allocator besides
(...skipping 31 matching lines...) Expand 10 before | Expand all | Expand 10 after
1081 } 1169 }
1082 } 1170 }
1083 1171
1084 // The default "do_free" that uses the default callback. 1172 // The default "do_free" that uses the default callback.
1085 inline void do_free(void* ptr) { 1173 inline void do_free(void* ptr) {
1086 return do_free_with_callback(ptr, &InvalidFree); 1174 return do_free_with_callback(ptr, &InvalidFree);
1087 } 1175 }
1088 1176
1089 // NOTE: some logic here is duplicated in GetOwnership (above), for 1177 // NOTE: some logic here is duplicated in GetOwnership (above), for
1090 // speed. If you change this function, look at that one too. 1178 // speed. If you change this function, look at that one too.
1091 inline size_t GetSizeWithCallback(void* ptr, 1179 inline size_t GetSizeWithCallback(const void* ptr,
1092 size_t (*invalid_getsize_fn)(void*)) { 1180 size_t (*invalid_getsize_fn)(const void*)) {
1093 if (ptr == NULL) 1181 if (ptr == NULL)
1094 return 0; 1182 return 0;
1095 const PageID p = reinterpret_cast<uintptr_t>(ptr) >> kPageShift; 1183 const PageID p = reinterpret_cast<uintptr_t>(ptr) >> kPageShift;
1096 size_t cl = Static::pageheap()->GetSizeClassIfCached(p); 1184 size_t cl = Static::pageheap()->GetSizeClassIfCached(p);
1097 if (cl != 0) { 1185 if (cl != 0) {
1098 return Static::sizemap()->ByteSizeForClass(cl); 1186 return Static::sizemap()->ByteSizeForClass(cl);
1099 } else { 1187 } else {
1100 const Span *span = Static::pageheap()->GetDescriptor(p); 1188 const Span *span = Static::pageheap()->GetDescriptor(p);
1101 if (span == NULL) { // means we do not own this memory 1189 if (span == NULL) { // means we do not own this memory
1102 return (*invalid_getsize_fn)(ptr); 1190 return (*invalid_getsize_fn)(ptr);
1103 } else if (span->sizeclass != 0) { 1191 } else if (span->sizeclass != 0) {
1104 Static::pageheap()->CacheSizeClass(p, span->sizeclass); 1192 Static::pageheap()->CacheSizeClass(p, span->sizeclass);
1105 return Static::sizemap()->ByteSizeForClass(span->sizeclass); 1193 return Static::sizemap()->ByteSizeForClass(span->sizeclass);
1106 } else { 1194 } else {
1107 return span->length << kPageShift; 1195 return span->length << kPageShift;
1108 } 1196 }
1109 } 1197 }
1110 } 1198 }
1111 1199
1112 // This lets you call back to a given function pointer if ptr is invalid. 1200 // This lets you call back to a given function pointer if ptr is invalid.
1113 // It is used primarily by windows code which wants a specialized callback. 1201 // It is used primarily by windows code which wants a specialized callback.
1114 inline void* do_realloc_with_callback( 1202 inline void* do_realloc_with_callback(
1115 void* old_ptr, size_t new_size, 1203 void* old_ptr, size_t new_size,
1116 void (*invalid_free_fn)(void*), 1204 void (*invalid_free_fn)(void*),
1117 size_t (*invalid_get_size_fn)(void*)) { 1205 size_t (*invalid_get_size_fn)(const void*)) {
1118 // Get the size of the old entry 1206 // Get the size of the old entry
1119 const size_t old_size = GetSizeWithCallback(old_ptr, invalid_get_size_fn); 1207 const size_t old_size = GetSizeWithCallback(old_ptr, invalid_get_size_fn);
1120 1208
1121 // Reallocate if the new size is larger than the old size, 1209 // Reallocate if the new size is larger than the old size,
1122 // or if the new size is significantly smaller than the old size. 1210 // or if the new size is significantly smaller than the old size.
1123 // We do hysteresis to avoid resizing ping-pongs: 1211 // We do hysteresis to avoid resizing ping-pongs:
1124 // . If we need to grow, grow to max(new_size, old_size * 1.X) 1212 // . If we need to grow, grow to max(new_size, old_size * 1.X)
1125 // . Don't shrink unless new_size < old_size * 0.Y 1213 // . Don't shrink unless new_size < old_size * 0.Y
1126 // X and Y trade-off time for wasted space. For now we do 1.25 and 0.5. 1214 // X and Y trade-off time for wasted space. For now we do 1.25 and 0.5.
1127 const int lower_bound_to_grow = old_size + old_size / 4; 1215 const int lower_bound_to_grow = old_size + old_size / 4;
(...skipping 120 matching lines...) Expand 10 before | Expand all | Expand 10 after
1248 PrintStats(1); 1336 PrintStats(1);
1249 } 1337 }
1250 1338
1251 inline int do_mallopt(int cmd, int value) { 1339 inline int do_mallopt(int cmd, int value) {
1252 return 1; // Indicates error 1340 return 1; // Indicates error
1253 } 1341 }
1254 1342
1255 #ifdef HAVE_STRUCT_MALLINFO 1343 #ifdef HAVE_STRUCT_MALLINFO
1256 inline struct mallinfo do_mallinfo() { 1344 inline struct mallinfo do_mallinfo() {
1257 TCMallocStats stats; 1345 TCMallocStats stats;
1258 ExtractStats(&stats, NULL); 1346 ExtractStats(&stats, NULL, NULL, NULL);
1259 1347
1260 // Just some of the fields are filled in. 1348 // Just some of the fields are filled in.
1261 struct mallinfo info; 1349 struct mallinfo info;
1262 memset(&info, 0, sizeof(info)); 1350 memset(&info, 0, sizeof(info));
1263 1351
1264 // Unfortunately, the struct contains "int" field, so some of the 1352 // Unfortunately, the struct contains "int" field, so some of the
1265 // size values will be truncated. 1353 // size values will be truncated.
1266 info.arena = static_cast<int>(stats.pageheap.system_bytes); 1354 info.arena = static_cast<int>(stats.pageheap.system_bytes);
1267 info.fsmblks = static_cast<int>(stats.thread_bytes 1355 info.fsmblks = static_cast<int>(stats.thread_bytes
1268 + stats.central_bytes 1356 + stats.central_bytes
(...skipping 103 matching lines...) Expand 10 before | Expand all | Expand 10 after
1372 } else { // allocation success 1460 } else { // allocation success
1373 return p; 1461 return p;
1374 } 1462 }
1375 #endif // PREANSINEW 1463 #endif // PREANSINEW
1376 } 1464 }
1377 } 1465 }
1378 1466
1379 } // end unnamed namespace 1467 } // end unnamed namespace
1380 1468
1381 // As promised, the definition of this function, declared above. 1469 // As promised, the definition of this function, declared above.
1382 size_t TCMallocImplementation::GetAllocatedSize(void* ptr) { 1470 size_t TCMallocImplementation::GetAllocatedSize(const void* ptr) {
1383 ASSERT(TCMallocImplementation::GetOwnership(ptr) 1471 ASSERT(TCMallocImplementation::GetOwnership(ptr)
1384 != TCMallocImplementation::kNotOwned); 1472 != TCMallocImplementation::kNotOwned);
1385 return GetSizeWithCallback(ptr, &InvalidGetAllocatedSize); 1473 return GetSizeWithCallback(ptr, &InvalidGetAllocatedSize);
1386 } 1474 }
1387 1475
1388 void TCMallocImplementation::MarkThreadBusy() { 1476 void TCMallocImplementation::MarkThreadBusy() {
1389 // Allocate to force the creation of a thread cache, but avoid 1477 // Allocate to force the creation of a thread cache, but avoid
1390 // invoking any hooks. 1478 // invoking any hooks.
1391 do_free(do_malloc(0)); 1479 do_free(do_malloc(0));
1392 } 1480 }
(...skipping 183 matching lines...) Expand 10 before | Expand all | Expand 10 after
1576 extern "C" PERFTOOLS_DLL_DECL struct mallinfo tc_mallinfo(void) __THROW { 1664 extern "C" PERFTOOLS_DLL_DECL struct mallinfo tc_mallinfo(void) __THROW {
1577 return do_mallinfo(); 1665 return do_mallinfo();
1578 } 1666 }
1579 #endif 1667 #endif
1580 1668
1581 extern "C" PERFTOOLS_DLL_DECL size_t tc_malloc_size(void* ptr) __THROW { 1669 extern "C" PERFTOOLS_DLL_DECL size_t tc_malloc_size(void* ptr) __THROW {
1582 return MallocExtension::instance()->GetAllocatedSize(ptr); 1670 return MallocExtension::instance()->GetAllocatedSize(ptr);
1583 } 1671 }
1584 1672
1585 #endif // TCMALLOC_USING_DEBUGALLOCATION 1673 #endif // TCMALLOC_USING_DEBUGALLOCATION
OLDNEW
« no previous file with comments | « third_party/tcmalloc/vendor/src/system-alloc.cc ('k') | third_party/tcmalloc/vendor/src/tests/current_allocated_bytes_test.cc » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698