Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(147)

Side by Side Diff: third_party/tcmalloc/vendor/src/tcmalloc.cc

Issue 9701040: Revert 126715 - Update the tcmalloc vendor branch to r144 (gperftools 2.0). (Closed) Base URL: svn://svn.chromium.org/chrome/trunk/src/
Patch Set: Created 8 years, 9 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch | Annotate | Revision Log
OLDNEW
1 // Copyright (c) 2005, Google Inc. 1 // Copyright (c) 2005, Google Inc.
2 // All rights reserved. 2 // All rights reserved.
3 // 3 //
4 // Redistribution and use in source and binary forms, with or without 4 // Redistribution and use in source and binary forms, with or without
5 // modification, are permitted provided that the following conditions are 5 // modification, are permitted provided that the following conditions are
6 // met: 6 // met:
7 // 7 //
8 // * Redistributions of source code must retain the above copyright 8 // * Redistributions of source code must retain the above copyright
9 // notice, this list of conditions and the following disclaimer. 9 // notice, this list of conditions and the following disclaimer.
10 // * Redistributions in binary form must reproduce the above 10 // * Redistributions in binary form must reproduce the above
(...skipping 69 matching lines...) Expand 10 before | Expand all | Expand 10 after
80 // TODO: Bias reclamation to larger addresses 80 // TODO: Bias reclamation to larger addresses
81 // TODO: implement mallinfo/mallopt 81 // TODO: implement mallinfo/mallopt
82 // TODO: Better testing 82 // TODO: Better testing
83 // 83 //
84 // 9/28/2003 (new page-level allocator replaces ptmalloc2): 84 // 9/28/2003 (new page-level allocator replaces ptmalloc2):
85 // * malloc/free of small objects goes from ~300 ns to ~50 ns. 85 // * malloc/free of small objects goes from ~300 ns to ~50 ns.
86 // * allocation of a reasonably complicated struct 86 // * allocation of a reasonably complicated struct
87 // goes from about 1100 ns to about 300 ns. 87 // goes from about 1100 ns to about 300 ns.
88 88
89 #include "config.h" 89 #include "config.h"
90 #include <gperftools/tcmalloc.h> 90 #include <google/tcmalloc.h>
91 91
92 #include <errno.h> // for ENOMEM, EINVAL, errno 92 #include <errno.h> // for ENOMEM, EINVAL, errno
93 #ifdef HAVE_SYS_CDEFS_H 93 #ifdef HAVE_SYS_CDEFS_H
94 #include <sys/cdefs.h> // for __THROW 94 #include <sys/cdefs.h> // for __THROW
95 #endif 95 #endif
96 #if defined HAVE_STDINT_H 96 #if defined HAVE_STDINT_H
97 #include <stdint.h> 97 #include <stdint.h>
98 #elif defined HAVE_INTTYPES_H 98 #elif defined HAVE_INTTYPES_H
99 #include <inttypes.h> 99 #include <inttypes.h>
100 #else 100 #else
101 #include <sys/types.h> 101 #include <sys/types.h>
102 #endif 102 #endif
103 #include <stddef.h> // for size_t, NULL 103 #include <stddef.h> // for size_t, NULL
104 #include <stdlib.h> // for getenv 104 #include <stdlib.h> // for getenv
105 #include <string.h> // for strcmp, memset, strlen, etc 105 #include <string.h> // for strcmp, memset, strlen, etc
106 #ifdef HAVE_UNISTD_H 106 #ifdef HAVE_UNISTD_H
107 #include <unistd.h> // for getpagesize, write, etc 107 #include <unistd.h> // for getpagesize, write, etc
108 #endif 108 #endif
109 #include <algorithm> // for max, min 109 #include <algorithm> // for max, min
110 #include <limits> // for numeric_limits 110 #include <limits> // for numeric_limits
111 #include <new> // for nothrow_t (ptr only), etc 111 #include <new> // for nothrow_t (ptr only), etc
112 #include <vector> // for vector 112 #include <vector> // for vector
113 113
114 #include <gperftools/malloc_extension.h> 114 #include <google/malloc_extension.h>
115 #include <gperftools/malloc_hook.h> // for MallocHook 115 #include <google/malloc_hook.h> // for MallocHook
116 #include "base/basictypes.h" // for int64 116 #include "base/basictypes.h" // for int64
117 #include "base/commandlineflags.h" // for RegisterFlagValidator, etc 117 #include "base/commandlineflags.h" // for RegisterFlagValidator, etc
118 #include "base/dynamic_annotations.h" // for RunningOnValgrind 118 #include "base/dynamic_annotations.h" // for RunningOnValgrind
119 #include "base/spinlock.h" // for SpinLockHolder 119 #include "base/spinlock.h" // for SpinLockHolder
120 #include "central_freelist.h" // for CentralFreeListPadded 120 #include "central_freelist.h" // for CentralFreeListPadded
121 #include "common.h" // for StackTrace, kPageShift, etc 121 #include "common.h" // for StackTrace, kPageShift, etc
122 #include "internal_logging.h" // for ASSERT, TCMalloc_Printer, etc 122 #include "internal_logging.h" // for ASSERT, TCMalloc_Printer, etc
123 #include "linked_list.h" // for SLL_SetNext 123 #include "linked_list.h" // for SLL_SetNext
124 #include "malloc_hook-inl.h" // for MallocHook::InvokeNewHook, etc 124 #include "malloc_hook-inl.h" // for MallocHook::InvokeNewHook, etc
125 #include "page_heap.h" // for PageHeap, PageHeap::Stats 125 #include "page_heap.h" // for PageHeap, PageHeap::Stats
(...skipping 14 matching lines...) Expand all
140 # include <sys/malloc.h> 140 # include <sys/malloc.h>
141 # elif defined(HAVE_MALLOC_MALLOC_H) 141 # elif defined(HAVE_MALLOC_MALLOC_H)
142 # include <malloc/malloc.h> 142 # include <malloc/malloc.h>
143 # endif 143 # endif
144 #endif 144 #endif
145 145
146 #if (defined(_WIN32) && !defined(__CYGWIN__) && !defined(__CYGWIN32__)) && !defi ned(WIN32_OVERRIDE_ALLOCATORS) 146 #if (defined(_WIN32) && !defined(__CYGWIN__) && !defined(__CYGWIN32__)) && !defi ned(WIN32_OVERRIDE_ALLOCATORS)
147 # define WIN32_DO_PATCHING 1 147 # define WIN32_DO_PATCHING 1
148 #endif 148 #endif
149 149
150 // Some windows file somewhere (at least on cygwin) #define's small (!)
151 #undef small
152
153 using STL_NAMESPACE::max; 150 using STL_NAMESPACE::max;
154 using STL_NAMESPACE::numeric_limits; 151 using STL_NAMESPACE::numeric_limits;
155 using STL_NAMESPACE::vector; 152 using STL_NAMESPACE::vector;
156 153
157 #include "libc_override.h" 154 #include "libc_override.h"
158 155
159 // __THROW is defined in glibc (via <sys/cdefs.h>). It means, 156 // __THROW is defined in glibc (via <sys/cdefs.h>). It means,
160 // counter-intuitively, "This function will never throw an exception." 157 // counter-intuitively, "This function will never throw an exception."
161 // It's an optional optimization tool, but we may need to use it to 158 // It's an optional optimization tool, but we may need to use it to
162 // match glibc prototypes. 159 // match glibc prototypes.
163 #ifndef __THROW // I guess we're not on a glibc system 160 #ifndef __THROW // I guess we're not on a glibc system
164 # define __THROW // __THROW is just an optimization, so ok to make it "" 161 # define __THROW // __THROW is just an optimization, so ok to make it ""
165 #endif 162 #endif
166 163
167 using tcmalloc::AlignmentForSize; 164 using tcmalloc::AlignmentForSize;
168 using tcmalloc::kLog;
169 using tcmalloc::kCrash;
170 using tcmalloc::kCrashWithStats;
171 using tcmalloc::Log;
172 using tcmalloc::PageHeap; 165 using tcmalloc::PageHeap;
173 using tcmalloc::PageHeapAllocator; 166 using tcmalloc::PageHeapAllocator;
174 using tcmalloc::SizeMap; 167 using tcmalloc::SizeMap;
175 using tcmalloc::Span; 168 using tcmalloc::Span;
176 using tcmalloc::StackTrace; 169 using tcmalloc::StackTrace;
177 using tcmalloc::Static; 170 using tcmalloc::Static;
178 using tcmalloc::ThreadCache; 171 using tcmalloc::ThreadCache;
179 172
180 DECLARE_int64(tcmalloc_sample_parameter); 173 DECLARE_int64(tcmalloc_sample_parameter);
181 DECLARE_double(tcmalloc_release_rate); 174 DECLARE_double(tcmalloc_release_rate);
(...skipping 97 matching lines...) Expand 10 before | Expand all | Expand 10 after
279 272
280 static int tc_new_mode = 0; // See tc_set_new_mode(). 273 static int tc_new_mode = 0; // See tc_set_new_mode().
281 274
282 // Routines such as free() and realloc() catch some erroneous pointers 275 // Routines such as free() and realloc() catch some erroneous pointers
283 // passed to them, and invoke the below when they do. (An erroneous pointer 276 // passed to them, and invoke the below when they do. (An erroneous pointer
284 // won't be caught if it's within a valid span or a stale span for which 277 // won't be caught if it's within a valid span or a stale span for which
285 // the pagemap cache has a non-zero sizeclass.) This is a cheap (source-editing 278 // the pagemap cache has a non-zero sizeclass.) This is a cheap (source-editing
286 // required) kind of exception handling for these routines. 279 // required) kind of exception handling for these routines.
287 namespace { 280 namespace {
288 void InvalidFree(void* ptr) { 281 void InvalidFree(void* ptr) {
289 Log(kCrash, __FILE__, __LINE__, "Attempt to free invalid pointer", ptr); 282 CRASH("Attempt to free invalid pointer: %p\n", ptr);
290 } 283 }
291 284
292 size_t InvalidGetSizeForRealloc(const void* old_ptr) { 285 size_t InvalidGetSizeForRealloc(void* old_ptr) {
293 Log(kCrash, __FILE__, __LINE__, 286 CRASH("Attempt to realloc invalid pointer: %p\n", old_ptr);
294 "Attempt to realloc invalid pointer", old_ptr);
295 return 0; 287 return 0;
296 } 288 }
297 289
298 size_t InvalidGetAllocatedSize(const void* ptr) { 290 size_t InvalidGetAllocatedSize(void* ptr) {
299 Log(kCrash, __FILE__, __LINE__, 291 CRASH("Attempt to get the size of an invalid pointer: %p\n", ptr);
300 "Attempt to get the size of an invalid pointer", ptr);
301 return 0; 292 return 0;
302 } 293 }
303 } // unnamed namespace 294 } // unnamed namespace
304 295
305 // Extract interesting stats 296 // Extract interesting stats
306 struct TCMallocStats { 297 struct TCMallocStats {
307 uint64_t thread_bytes; // Bytes in thread caches 298 uint64_t thread_bytes; // Bytes in thread caches
308 uint64_t central_bytes; // Bytes in central cache 299 uint64_t central_bytes; // Bytes in central cache
309 uint64_t transfer_bytes; // Bytes in central transfer cache 300 uint64_t transfer_bytes; // Bytes in central transfer cache
310 uint64_t metadata_bytes; // Bytes alloced for metadata 301 uint64_t metadata_bytes; // Bytes alloced for metadata
311 PageHeap::Stats pageheap; // Stats from page heap 302 PageHeap::Stats pageheap; // Stats from page heap
312 }; 303 };
313 304
314 // Get stats into "r". Also get per-size-class counts if class_count != NULL 305 // Get stats into "r". Also get per-size-class counts if class_count != NULL
315 static void ExtractStats(TCMallocStats* r, uint64_t* class_count, 306 static void ExtractStats(TCMallocStats* r, uint64_t* class_count) {
316 PageHeap::SmallSpanStats* small_spans,
317 PageHeap::LargeSpanStats* large_spans) {
318 r->central_bytes = 0; 307 r->central_bytes = 0;
319 r->transfer_bytes = 0; 308 r->transfer_bytes = 0;
320 for (int cl = 0; cl < kNumClasses; ++cl) { 309 for (int cl = 0; cl < kNumClasses; ++cl) {
321 const int length = Static::central_cache()[cl].length(); 310 const int length = Static::central_cache()[cl].length();
322 const int tc_length = Static::central_cache()[cl].tc_length(); 311 const int tc_length = Static::central_cache()[cl].tc_length();
323 const size_t cache_overhead = Static::central_cache()[cl].OverheadBytes(); 312 const size_t cache_overhead = Static::central_cache()[cl].OverheadBytes();
324 const size_t size = static_cast<uint64_t>( 313 const size_t size = static_cast<uint64_t>(
325 Static::sizemap()->ByteSizeForClass(cl)); 314 Static::sizemap()->ByteSizeForClass(cl));
326 r->central_bytes += (size * length) + cache_overhead; 315 r->central_bytes += (size * length) + cache_overhead;
327 r->transfer_bytes += (size * tc_length); 316 r->transfer_bytes += (size * tc_length);
328 if (class_count) class_count[cl] = length + tc_length; 317 if (class_count) class_count[cl] = length + tc_length;
329 } 318 }
330 319
331 // Add stats from per-thread heaps 320 // Add stats from per-thread heaps
332 r->thread_bytes = 0; 321 r->thread_bytes = 0;
333 { // scope 322 { // scope
334 SpinLockHolder h(Static::pageheap_lock()); 323 SpinLockHolder h(Static::pageheap_lock());
335 ThreadCache::GetThreadStats(&r->thread_bytes, class_count); 324 ThreadCache::GetThreadStats(&r->thread_bytes, class_count);
336 r->metadata_bytes = tcmalloc::metadata_system_bytes(); 325 r->metadata_bytes = tcmalloc::metadata_system_bytes();
337 r->pageheap = Static::pageheap()->stats(); 326 r->pageheap = Static::pageheap()->stats();
338 if (small_spans != NULL) {
339 Static::pageheap()->GetSmallSpanStats(small_spans);
340 }
341 if (large_spans != NULL) {
342 Static::pageheap()->GetLargeSpanStats(large_spans);
343 }
344 } 327 }
345 } 328 }
346 329
347 static double PagesToMiB(uint64_t pages) {
348 return (pages << kPageShift) / 1048576.0;
349 }
350
351 // WRITE stats to "out" 330 // WRITE stats to "out"
352 static void DumpStats(TCMalloc_Printer* out, int level) { 331 static void DumpStats(TCMalloc_Printer* out, int level) {
353 TCMallocStats stats; 332 TCMallocStats stats;
354 uint64_t class_count[kNumClasses]; 333 uint64_t class_count[kNumClasses];
355 PageHeap::SmallSpanStats small; 334 ExtractStats(&stats, (level >= 2 ? class_count : NULL));
356 PageHeap::LargeSpanStats large;
357 if (level >= 2) {
358 ExtractStats(&stats, class_count, &small, &large);
359 } else {
360 ExtractStats(&stats, NULL, NULL, NULL);
361 }
362 335
363 static const double MiB = 1048576.0; 336 static const double MiB = 1048576.0;
364 337
365 const uint64_t virtual_memory_used = (stats.pageheap.system_bytes 338 const uint64_t virtual_memory_used = (stats.pageheap.system_bytes
366 + stats.metadata_bytes); 339 + stats.metadata_bytes);
367 const uint64_t physical_memory_used = (virtual_memory_used 340 const uint64_t physical_memory_used = (virtual_memory_used
368 - stats.pageheap.unmapped_bytes); 341 - stats.pageheap.unmapped_bytes);
369 const uint64_t bytes_in_use_by_app = (physical_memory_used 342 const uint64_t bytes_in_use_by_app = (physical_memory_used
370 - stats.metadata_bytes 343 - stats.metadata_bytes
371 - stats.pageheap.free_bytes 344 - stats.pageheap.free_bytes
(...skipping 52 matching lines...) Expand 10 before | Expand all | Expand 10 after
424 cumulative += class_bytes; 397 cumulative += class_bytes;
425 out->printf("class %3d [ %8" PRIuS " bytes ] : " 398 out->printf("class %3d [ %8" PRIuS " bytes ] : "
426 "%8" PRIu64 " objs; %5.1f MiB; %5.1f cum MiB\n", 399 "%8" PRIu64 " objs; %5.1f MiB; %5.1f cum MiB\n",
427 cl, Static::sizemap()->ByteSizeForClass(cl), 400 cl, Static::sizemap()->ByteSizeForClass(cl),
428 class_count[cl], 401 class_count[cl],
429 class_bytes / MiB, 402 class_bytes / MiB,
430 cumulative / MiB); 403 cumulative / MiB);
431 } 404 }
432 } 405 }
433 406
434 // append page heap info 407 SpinLockHolder h(Static::pageheap_lock());
435 int nonempty_sizes = 0; 408 Static::pageheap()->Dump(out);
436 for (int s = 0; s < kMaxPages; s++) {
437 if (small.normal_length[s] + small.returned_length[s] > 0) {
438 nonempty_sizes++;
439 }
440 }
441 out->printf("------------------------------------------------\n");
442 out->printf("PageHeap: %d sizes; %6.1f MiB free; %6.1f MiB unmapped\n",
443 nonempty_sizes, stats.pageheap.free_bytes / MiB,
444 stats.pageheap.unmapped_bytes / MiB);
445 out->printf("------------------------------------------------\n");
446 uint64_t total_normal = 0;
447 uint64_t total_returned = 0;
448 for (int s = 0; s < kMaxPages; s++) {
449 const int n_length = small.normal_length[s];
450 const int r_length = small.returned_length[s];
451 if (n_length + r_length > 0) {
452 uint64_t n_pages = s * n_length;
453 uint64_t r_pages = s * r_length;
454 total_normal += n_pages;
455 total_returned += r_pages;
456 out->printf("%6u pages * %6u spans ~ %6.1f MiB; %6.1f MiB cum"
457 "; unmapped: %6.1f MiB; %6.1f MiB cum\n",
458 s,
459 (n_length + r_length),
460 PagesToMiB(n_pages + r_pages),
461 PagesToMiB(total_normal + total_returned),
462 PagesToMiB(r_pages),
463 PagesToMiB(total_returned));
464 }
465 }
466
467 total_normal += large.normal_pages;
468 total_returned += large.returned_pages;
469 out->printf(">255 large * %6u spans ~ %6.1f MiB; %6.1f MiB cum"
470 "; unmapped: %6.1f MiB; %6.1f MiB cum\n",
471 static_cast<unsigned int>(large.spans),
472 PagesToMiB(large.normal_pages + large.returned_pages),
473 PagesToMiB(total_normal + total_returned),
474 PagesToMiB(large.returned_pages),
475 PagesToMiB(total_returned));
476 } 409 }
477 } 410 }
478 411
479 static void PrintStats(int level) { 412 static void PrintStats(int level) {
480 const int kBufferSize = 16 << 10; 413 const int kBufferSize = 16 << 10;
481 char* buffer = new char[kBufferSize]; 414 char* buffer = new char[kBufferSize];
482 TCMalloc_Printer printer(buffer, kBufferSize); 415 TCMalloc_Printer printer(buffer, kBufferSize);
483 DumpStats(&printer, level); 416 DumpStats(&printer, level);
484 write(STDERR_FILENO, buffer, strlen(buffer)); 417 write(STDERR_FILENO, buffer, strlen(buffer));
485 delete[] buffer; 418 delete[] buffer;
486 } 419 }
487 420
488 static void** DumpHeapGrowthStackTraces() { 421 static void** DumpHeapGrowthStackTraces() {
489 // Count how much space we need 422 // Count how much space we need
490 int needed_slots = 0; 423 int needed_slots = 0;
491 { 424 {
492 SpinLockHolder h(Static::pageheap_lock()); 425 SpinLockHolder h(Static::pageheap_lock());
493 for (StackTrace* t = Static::growth_stacks(); 426 for (StackTrace* t = Static::growth_stacks();
494 t != NULL; 427 t != NULL;
495 t = reinterpret_cast<StackTrace*>( 428 t = reinterpret_cast<StackTrace*>(
496 t->stack[tcmalloc::kMaxStackDepth-1])) { 429 t->stack[tcmalloc::kMaxStackDepth-1])) {
497 needed_slots += 3 + t->depth; 430 needed_slots += 3 + t->depth;
498 } 431 }
499 needed_slots += 100; // Slop in case list grows 432 needed_slots += 100; // Slop in case list grows
500 needed_slots += needed_slots/8; // An extra 12.5% slop 433 needed_slots += needed_slots/8; // An extra 12.5% slop
501 } 434 }
502 435
503 void** result = new void*[needed_slots]; 436 void** result = new void*[needed_slots];
504 if (result == NULL) { 437 if (result == NULL) {
505 Log(kLog, __FILE__, __LINE__, 438 MESSAGE("tcmalloc: allocation failed for stack trace slots",
506 "tcmalloc: allocation failed for stack trace slots", 439 needed_slots * sizeof(*result));
507 needed_slots * sizeof(*result));
508 return NULL; 440 return NULL;
509 } 441 }
510 442
511 SpinLockHolder h(Static::pageheap_lock()); 443 SpinLockHolder h(Static::pageheap_lock());
512 int used_slots = 0; 444 int used_slots = 0;
513 for (StackTrace* t = Static::growth_stacks(); 445 for (StackTrace* t = Static::growth_stacks();
514 t != NULL; 446 t != NULL;
515 t = reinterpret_cast<StackTrace*>( 447 t = reinterpret_cast<StackTrace*>(
516 t->stack[tcmalloc::kMaxStackDepth-1])) { 448 t->stack[tcmalloc::kMaxStackDepth-1])) {
517 ASSERT(used_slots < needed_slots); // Need to leave room for terminator 449 ASSERT(used_slots < needed_slots); // Need to leave room for terminator
(...skipping 105 matching lines...) Expand 10 before | Expand all | Expand 10 after
623 555
624 virtual void Ranges(void* arg, RangeFunction func) { 556 virtual void Ranges(void* arg, RangeFunction func) {
625 IterateOverRanges(arg, func); 557 IterateOverRanges(arg, func);
626 } 558 }
627 559
628 virtual bool GetNumericProperty(const char* name, size_t* value) { 560 virtual bool GetNumericProperty(const char* name, size_t* value) {
629 ASSERT(name != NULL); 561 ASSERT(name != NULL);
630 562
631 if (strcmp(name, "generic.current_allocated_bytes") == 0) { 563 if (strcmp(name, "generic.current_allocated_bytes") == 0) {
632 TCMallocStats stats; 564 TCMallocStats stats;
633 ExtractStats(&stats, NULL, NULL, NULL); 565 ExtractStats(&stats, NULL);
634 *value = stats.pageheap.system_bytes 566 *value = stats.pageheap.system_bytes
635 - stats.thread_bytes 567 - stats.thread_bytes
636 - stats.central_bytes 568 - stats.central_bytes
637 - stats.transfer_bytes 569 - stats.transfer_bytes
638 - stats.pageheap.free_bytes 570 - stats.pageheap.free_bytes
639 - stats.pageheap.unmapped_bytes; 571 - stats.pageheap.unmapped_bytes;
640 return true; 572 return true;
641 } 573 }
642 574
643 if (strcmp(name, "generic.heap_size") == 0) { 575 if (strcmp(name, "generic.heap_size") == 0) {
644 TCMallocStats stats; 576 TCMallocStats stats;
645 ExtractStats(&stats, NULL, NULL, NULL); 577 ExtractStats(&stats, NULL);
646 *value = stats.pageheap.system_bytes; 578 *value = stats.pageheap.system_bytes;
647 return true; 579 return true;
648 } 580 }
649 581
650 if (strcmp(name, "tcmalloc.slack_bytes") == 0) { 582 if (strcmp(name, "tcmalloc.slack_bytes") == 0) {
651 // Kept for backwards compatibility. Now defined externally as: 583 // Kept for backwards compatibility. Now defined externally as:
652 // pageheap_free_bytes + pageheap_unmapped_bytes. 584 // pageheap_free_bytes + pageheap_unmapped_bytes.
653 SpinLockHolder l(Static::pageheap_lock()); 585 SpinLockHolder l(Static::pageheap_lock());
654 PageHeap::Stats stats = Static::pageheap()->stats(); 586 PageHeap::Stats stats = Static::pageheap()->stats();
655 *value = stats.free_bytes + stats.unmapped_bytes; 587 *value = stats.free_bytes + stats.unmapped_bytes;
(...skipping 13 matching lines...) Expand all
669 } 601 }
670 602
671 if (strcmp(name, "tcmalloc.max_total_thread_cache_bytes") == 0) { 603 if (strcmp(name, "tcmalloc.max_total_thread_cache_bytes") == 0) {
672 SpinLockHolder l(Static::pageheap_lock()); 604 SpinLockHolder l(Static::pageheap_lock());
673 *value = ThreadCache::overall_thread_cache_size(); 605 *value = ThreadCache::overall_thread_cache_size();
674 return true; 606 return true;
675 } 607 }
676 608
677 if (strcmp(name, "tcmalloc.current_total_thread_cache_bytes") == 0) { 609 if (strcmp(name, "tcmalloc.current_total_thread_cache_bytes") == 0) {
678 TCMallocStats stats; 610 TCMallocStats stats;
679 ExtractStats(&stats, NULL, NULL, NULL); 611 ExtractStats(&stats, NULL);
680 *value = stats.thread_bytes; 612 *value = stats.thread_bytes;
681 return true; 613 return true;
682 } 614 }
683 615
684 return false; 616 return false;
685 } 617 }
686 618
687 virtual bool SetNumericProperty(const char* name, size_t value) { 619 virtual bool SetNumericProperty(const char* name, size_t value) {
688 ASSERT(name != NULL); 620 ASSERT(name != NULL);
689 621
(...skipping 60 matching lines...) Expand 10 before | Expand all | Expand 10 after
750 const size_t alloc_size = Static::sizemap()->ByteSizeForClass(cl); 682 const size_t alloc_size = Static::sizemap()->ByteSizeForClass(cl);
751 return alloc_size; 683 return alloc_size;
752 } else { 684 } else {
753 return tcmalloc::pages(size) << kPageShift; 685 return tcmalloc::pages(size) << kPageShift;
754 } 686 }
755 } 687 }
756 688
757 // This just calls GetSizeWithCallback, but because that's in an 689 // This just calls GetSizeWithCallback, but because that's in an
758 // unnamed namespace, we need to move the definition below it in the 690 // unnamed namespace, we need to move the definition below it in the
759 // file. 691 // file.
760 virtual size_t GetAllocatedSize(const void* ptr); 692 virtual size_t GetAllocatedSize(void* ptr);
761 693
762 // This duplicates some of the logic in GetSizeWithCallback, but is 694 // This duplicates some of the logic in GetSizeWithCallback, but is
763 // faster. This is important on OS X, where this function is called 695 // faster. This is important on OS X, where this function is called
764 // on every allocation operation. 696 // on every allocation operation.
765 virtual Ownership GetOwnership(const void* ptr) { 697 virtual Ownership GetOwnership(const void* ptr) {
766 const PageID p = reinterpret_cast<uintptr_t>(ptr) >> kPageShift; 698 const PageID p = reinterpret_cast<uintptr_t>(ptr) >> kPageShift;
767 // The rest of tcmalloc assumes that all allocated pointers use at 699 // The rest of tcmalloc assumes that all allocated pointers use at
768 // most kAddressBits bits. If ptr doesn't, then it definitely 700 // most kAddressBits bits. If ptr doesn't, then it definitely
769 // wasn't alloacted by tcmalloc. 701 // wasn't alloacted by tcmalloc.
770 if ((p >> (kAddressBits - kPageShift)) > 0) { 702 if ((p >> (kAddressBits - kPageShift)) > 0) {
(...skipping 53 matching lines...) Expand 10 before | Expand all | Expand 10 after
824 MallocExtension::FreeListInfo i; 756 MallocExtension::FreeListInfo i;
825 i.min_object_size = prev_class_size + 1; 757 i.min_object_size = prev_class_size + 1;
826 i.max_object_size = Static::sizemap()->ByteSizeForClass(cl); 758 i.max_object_size = Static::sizemap()->ByteSizeForClass(cl);
827 i.total_bytes_free = 759 i.total_bytes_free =
828 class_count[cl] * Static::sizemap()->ByteSizeForClass(cl); 760 class_count[cl] * Static::sizemap()->ByteSizeForClass(cl);
829 i.type = kThreadCacheType; 761 i.type = kThreadCacheType;
830 v->push_back(i); 762 v->push_back(i);
831 } 763 }
832 764
833 // append page heap info 765 // append page heap info
834 PageHeap::SmallSpanStats small; 766 int64 page_count_normal[kMaxPages];
835 PageHeap::LargeSpanStats large; 767 int64 page_count_returned[kMaxPages];
768 int64 span_count_normal;
769 int64 span_count_returned;
836 { 770 {
837 SpinLockHolder h(Static::pageheap_lock()); 771 SpinLockHolder h(Static::pageheap_lock());
838 Static::pageheap()->GetSmallSpanStats(&small); 772 Static::pageheap()->GetClassSizes(page_count_normal,
839 Static::pageheap()->GetLargeSpanStats(&large); 773 page_count_returned,
774 &span_count_normal,
775 &span_count_returned);
840 } 776 }
841 777
842 // large spans: mapped 778 // spans: mapped
843 MallocExtension::FreeListInfo span_info; 779 MallocExtension::FreeListInfo span_info;
844 span_info.type = kLargeSpanType; 780 span_info.type = kLargeSpanType;
845 span_info.max_object_size = (numeric_limits<size_t>::max)(); 781 span_info.max_object_size = (numeric_limits<size_t>::max)();
846 span_info.min_object_size = kMaxPages << kPageShift; 782 span_info.min_object_size = kMaxPages << kPageShift;
847 span_info.total_bytes_free = large.normal_pages << kPageShift; 783 span_info.total_bytes_free = span_count_normal << kPageShift;
848 v->push_back(span_info); 784 v->push_back(span_info);
849 785
850 // large spans: unmapped 786 // spans: unmapped
851 span_info.type = kLargeUnmappedSpanType; 787 span_info.type = kLargeUnmappedSpanType;
852 span_info.total_bytes_free = large.returned_pages << kPageShift; 788 span_info.total_bytes_free = span_count_returned << kPageShift;
853 v->push_back(span_info); 789 v->push_back(span_info);
854 790
855 // small spans
856 for (int s = 1; s < kMaxPages; s++) { 791 for (int s = 1; s < kMaxPages; s++) {
857 MallocExtension::FreeListInfo i; 792 MallocExtension::FreeListInfo i;
858 i.max_object_size = (s << kPageShift); 793 i.max_object_size = (s << kPageShift);
859 i.min_object_size = ((s - 1) << kPageShift); 794 i.min_object_size = ((s - 1) << kPageShift);
860 795
861 i.type = kPageHeapType; 796 i.type = kPageHeapType;
862 i.total_bytes_free = (s << kPageShift) * small.normal_length[s]; 797 i.total_bytes_free = (s << kPageShift) * page_count_normal[s];
863 v->push_back(i); 798 v->push_back(i);
864 799
865 i.type = kPageHeapUnmappedType; 800 i.type = kPageHeapUnmappedType;
866 i.total_bytes_free = (s << kPageShift) * small.returned_length[s]; 801 i.total_bytes_free = (s << kPageShift) * page_count_returned[s];
867 v->push_back(i); 802 v->push_back(i);
868 } 803 }
869 } 804 }
870 }; 805 };
871 806
872 // The constructor allocates an object to ensure that initialization 807 // The constructor allocates an object to ensure that initialization
873 // runs before main(), and therefore we do not have a chance to become 808 // runs before main(), and therefore we do not have a chance to become
874 // multi-threaded before initialization. We also create the TSD key 809 // multi-threaded before initialization. We also create the TSD key
875 // here. Presumably by the time this constructor runs, glibc is in 810 // here. Presumably by the time this constructor runs, glibc is in
876 // good enough shape to handle pthread_key_create(). 811 // good enough shape to handle pthread_key_create().
877 // 812 //
878 // The constructor also takes the opportunity to tell STL to use 813 // The constructor also takes the opportunity to tell STL to use
879 // tcmalloc. We want to do this early, before construct time, so 814 // tcmalloc. We want to do this early, before construct time, so
880 // all user STL allocations go through tcmalloc (which works really 815 // all user STL allocations go through tcmalloc (which works really
881 // well for STL). 816 // well for STL).
882 // 817 //
883 // The destructor prints stats when the program exits. 818 // The destructor prints stats when the program exits.
884 static int tcmallocguard_refcount = 0; // no lock needed: runs before main() 819 static int tcmallocguard_refcount = 0; // no lock needed: runs before main()
885 TCMallocGuard::TCMallocGuard() { 820 TCMallocGuard::TCMallocGuard() {
886 if (tcmallocguard_refcount++ == 0) { 821 if (tcmallocguard_refcount++ == 0) {
887 #ifdef HAVE_TLS // this is true if the cc/ld/libc combo support TLS 822 #ifdef HAVE_TLS // this is true if the cc/ld/libc combo support TLS
888 // Check whether the kernel also supports TLS (needs to happen at runtime) 823 // Check whether the kernel also supports TLS (needs to happen at runtime)
889 tcmalloc::CheckIfKernelSupportsTLS(); 824 tcmalloc::CheckIfKernelSupportsTLS();
890 #endif 825 #endif
891 ReplaceSystemAlloc(); // defined in libc_override_*.h 826 ReplaceSystemAlloc(); // defined in libc_override_*.h
827 #if defined(__APPLE__)
828 // To break the recursive call of malloc, as malloc -> TCMALLOC_MESSAGE
829 // -> snprintf -> localeconv_l -> malloc, on MacOS.
830 char buf[32];
831 snprintf(buf, sizeof(buf), "%d", tcmallocguard_refcount);
832 #endif
892 tc_free(tc_malloc(1)); 833 tc_free(tc_malloc(1));
893 ThreadCache::InitTSD(); 834 ThreadCache::InitTSD();
894 tc_free(tc_malloc(1)); 835 tc_free(tc_malloc(1));
895 // Either we, or debugallocation.cc, or valgrind will control memory 836 // Either we, or debugallocation.cc, or valgrind will control memory
896 // management. We register our extension if we're the winner. 837 // management. We register our extension if we're the winner.
897 #ifdef TCMALLOC_USING_DEBUGALLOCATION 838 #ifdef TCMALLOC_USING_DEBUGALLOCATION
898 // Let debugallocation register its extension. 839 // Let debugallocation register its extension.
899 #else 840 #else
900 if (RunningOnValgrind()) { 841 if (RunningOnValgrind()) {
901 // Let Valgrind uses its own malloc (so don't register our extension). 842 // Let Valgrind uses its own malloc (so don't register our extension).
(...skipping 186 matching lines...) Expand 10 before | Expand all | Expand 10 after
1088 1029
1089 static inline ThreadCache* GetCacheIfPresent() { 1030 static inline ThreadCache* GetCacheIfPresent() {
1090 void* const p = ThreadCache::GetCacheIfPresent(); 1031 void* const p = ThreadCache::GetCacheIfPresent();
1091 return reinterpret_cast<ThreadCache*>(p); 1032 return reinterpret_cast<ThreadCache*>(p);
1092 } 1033 }
1093 1034
1094 // This lets you call back to a given function pointer if ptr is invalid. 1035 // This lets you call back to a given function pointer if ptr is invalid.
1095 // It is used primarily by windows code which wants a specialized callback. 1036 // It is used primarily by windows code which wants a specialized callback.
1096 inline void do_free_with_callback(void* ptr, void (*invalid_free_fn)(void*)) { 1037 inline void do_free_with_callback(void* ptr, void (*invalid_free_fn)(void*)) {
1097 if (ptr == NULL) return; 1038 if (ptr == NULL) return;
1098 if (Static::pageheap() == NULL) { 1039 ASSERT(Static::pageheap() != NULL); // Should not call free() before malloc()
1099 // We called free() before malloc(). This can occur if the
1100 // (system) malloc() is called before tcmalloc is loaded, and then
1101 // free() is called after tcmalloc is loaded (and tc_free has
1102 // replaced free), but before the global constructor has run that
1103 // sets up the tcmalloc data structures.
1104 (*invalid_free_fn)(ptr); // Decide how to handle the bad free request
1105 return;
1106 }
1107 const PageID p = reinterpret_cast<uintptr_t>(ptr) >> kPageShift; 1040 const PageID p = reinterpret_cast<uintptr_t>(ptr) >> kPageShift;
1108 Span* span = NULL; 1041 Span* span = NULL;
1109 size_t cl = Static::pageheap()->GetSizeClassIfCached(p); 1042 size_t cl = Static::pageheap()->GetSizeClassIfCached(p);
1110 1043
1111 if (cl == 0) { 1044 if (cl == 0) {
1112 span = Static::pageheap()->GetDescriptor(p); 1045 span = Static::pageheap()->GetDescriptor(p);
1113 if (!span) { 1046 if (!span) {
1114 // span can be NULL because the pointer passed in is invalid 1047 // span can be NULL because the pointer passed in is invalid
1115 // (not something returned by malloc or friends), or because the 1048 // (not something returned by malloc or friends), or because the
1116 // pointer was allocated with some other allocator besides 1049 // pointer was allocated with some other allocator besides
(...skipping 31 matching lines...) Expand 10 before | Expand all | Expand 10 after
1148 } 1081 }
1149 } 1082 }
1150 1083
1151 // The default "do_free" that uses the default callback. 1084 // The default "do_free" that uses the default callback.
1152 inline void do_free(void* ptr) { 1085 inline void do_free(void* ptr) {
1153 return do_free_with_callback(ptr, &InvalidFree); 1086 return do_free_with_callback(ptr, &InvalidFree);
1154 } 1087 }
1155 1088
1156 // NOTE: some logic here is duplicated in GetOwnership (above), for 1089 // NOTE: some logic here is duplicated in GetOwnership (above), for
1157 // speed. If you change this function, look at that one too. 1090 // speed. If you change this function, look at that one too.
1158 inline size_t GetSizeWithCallback(const void* ptr, 1091 inline size_t GetSizeWithCallback(void* ptr,
1159 size_t (*invalid_getsize_fn)(const void*)) { 1092 size_t (*invalid_getsize_fn)(void*)) {
1160 if (ptr == NULL) 1093 if (ptr == NULL)
1161 return 0; 1094 return 0;
1162 const PageID p = reinterpret_cast<uintptr_t>(ptr) >> kPageShift; 1095 const PageID p = reinterpret_cast<uintptr_t>(ptr) >> kPageShift;
1163 size_t cl = Static::pageheap()->GetSizeClassIfCached(p); 1096 size_t cl = Static::pageheap()->GetSizeClassIfCached(p);
1164 if (cl != 0) { 1097 if (cl != 0) {
1165 return Static::sizemap()->ByteSizeForClass(cl); 1098 return Static::sizemap()->ByteSizeForClass(cl);
1166 } else { 1099 } else {
1167 const Span *span = Static::pageheap()->GetDescriptor(p); 1100 const Span *span = Static::pageheap()->GetDescriptor(p);
1168 if (span == NULL) { // means we do not own this memory 1101 if (span == NULL) { // means we do not own this memory
1169 return (*invalid_getsize_fn)(ptr); 1102 return (*invalid_getsize_fn)(ptr);
1170 } else if (span->sizeclass != 0) { 1103 } else if (span->sizeclass != 0) {
1171 Static::pageheap()->CacheSizeClass(p, span->sizeclass); 1104 Static::pageheap()->CacheSizeClass(p, span->sizeclass);
1172 return Static::sizemap()->ByteSizeForClass(span->sizeclass); 1105 return Static::sizemap()->ByteSizeForClass(span->sizeclass);
1173 } else { 1106 } else {
1174 return span->length << kPageShift; 1107 return span->length << kPageShift;
1175 } 1108 }
1176 } 1109 }
1177 } 1110 }
1178 1111
1179 // This lets you call back to a given function pointer if ptr is invalid. 1112 // This lets you call back to a given function pointer if ptr is invalid.
1180 // It is used primarily by windows code which wants a specialized callback. 1113 // It is used primarily by windows code which wants a specialized callback.
1181 inline void* do_realloc_with_callback( 1114 inline void* do_realloc_with_callback(
1182 void* old_ptr, size_t new_size, 1115 void* old_ptr, size_t new_size,
1183 void (*invalid_free_fn)(void*), 1116 void (*invalid_free_fn)(void*),
1184 size_t (*invalid_get_size_fn)(const void*)) { 1117 size_t (*invalid_get_size_fn)(void*)) {
1185 // Get the size of the old entry 1118 // Get the size of the old entry
1186 const size_t old_size = GetSizeWithCallback(old_ptr, invalid_get_size_fn); 1119 const size_t old_size = GetSizeWithCallback(old_ptr, invalid_get_size_fn);
1187 1120
1188 // Reallocate if the new size is larger than the old size, 1121 // Reallocate if the new size is larger than the old size,
1189 // or if the new size is significantly smaller than the old size. 1122 // or if the new size is significantly smaller than the old size.
1190 // We do hysteresis to avoid resizing ping-pongs: 1123 // We do hysteresis to avoid resizing ping-pongs:
1191 // . If we need to grow, grow to max(new_size, old_size * 1.X) 1124 // . If we need to grow, grow to max(new_size, old_size * 1.X)
1192 // . Don't shrink unless new_size < old_size * 0.Y 1125 // . Don't shrink unless new_size < old_size * 0.Y
1193 // X and Y trade-off time for wasted space. For now we do 1.25 and 0.5. 1126 // X and Y trade-off time for wasted space. For now we do 1.25 and 0.5.
1194 const int lower_bound_to_grow = old_size + old_size / 4; 1127 const int lower_bound_to_grow = old_size + old_size / 4;
(...skipping 120 matching lines...) Expand 10 before | Expand all | Expand 10 after
1315 PrintStats(1); 1248 PrintStats(1);
1316 } 1249 }
1317 1250
1318 inline int do_mallopt(int cmd, int value) { 1251 inline int do_mallopt(int cmd, int value) {
1319 return 1; // Indicates error 1252 return 1; // Indicates error
1320 } 1253 }
1321 1254
1322 #ifdef HAVE_STRUCT_MALLINFO 1255 #ifdef HAVE_STRUCT_MALLINFO
1323 inline struct mallinfo do_mallinfo() { 1256 inline struct mallinfo do_mallinfo() {
1324 TCMallocStats stats; 1257 TCMallocStats stats;
1325 ExtractStats(&stats, NULL, NULL, NULL); 1258 ExtractStats(&stats, NULL);
1326 1259
1327 // Just some of the fields are filled in. 1260 // Just some of the fields are filled in.
1328 struct mallinfo info; 1261 struct mallinfo info;
1329 memset(&info, 0, sizeof(info)); 1262 memset(&info, 0, sizeof(info));
1330 1263
1331 // Unfortunately, the struct contains "int" field, so some of the 1264 // Unfortunately, the struct contains "int" field, so some of the
1332 // size values will be truncated. 1265 // size values will be truncated.
1333 info.arena = static_cast<int>(stats.pageheap.system_bytes); 1266 info.arena = static_cast<int>(stats.pageheap.system_bytes);
1334 info.fsmblks = static_cast<int>(stats.thread_bytes 1267 info.fsmblks = static_cast<int>(stats.thread_bytes
1335 + stats.central_bytes 1268 + stats.central_bytes
(...skipping 103 matching lines...) Expand 10 before | Expand all | Expand 10 after
1439 } else { // allocation success 1372 } else { // allocation success
1440 return p; 1373 return p;
1441 } 1374 }
1442 #endif // PREANSINEW 1375 #endif // PREANSINEW
1443 } 1376 }
1444 } 1377 }
1445 1378
1446 } // end unnamed namespace 1379 } // end unnamed namespace
1447 1380
1448 // As promised, the definition of this function, declared above. 1381 // As promised, the definition of this function, declared above.
1449 size_t TCMallocImplementation::GetAllocatedSize(const void* ptr) { 1382 size_t TCMallocImplementation::GetAllocatedSize(void* ptr) {
1450 ASSERT(TCMallocImplementation::GetOwnership(ptr) 1383 ASSERT(TCMallocImplementation::GetOwnership(ptr)
1451 != TCMallocImplementation::kNotOwned); 1384 != TCMallocImplementation::kNotOwned);
1452 return GetSizeWithCallback(ptr, &InvalidGetAllocatedSize); 1385 return GetSizeWithCallback(ptr, &InvalidGetAllocatedSize);
1453 } 1386 }
1454 1387
1455 void TCMallocImplementation::MarkThreadBusy() { 1388 void TCMallocImplementation::MarkThreadBusy() {
1456 // Allocate to force the creation of a thread cache, but avoid 1389 // Allocate to force the creation of a thread cache, but avoid
1457 // invoking any hooks. 1390 // invoking any hooks.
1458 do_free(do_malloc(0)); 1391 do_free(do_malloc(0));
1459 } 1392 }
(...skipping 183 matching lines...) Expand 10 before | Expand all | Expand 10 after
1643 extern "C" PERFTOOLS_DLL_DECL struct mallinfo tc_mallinfo(void) __THROW { 1576 extern "C" PERFTOOLS_DLL_DECL struct mallinfo tc_mallinfo(void) __THROW {
1644 return do_mallinfo(); 1577 return do_mallinfo();
1645 } 1578 }
1646 #endif 1579 #endif
1647 1580
1648 extern "C" PERFTOOLS_DLL_DECL size_t tc_malloc_size(void* ptr) __THROW { 1581 extern "C" PERFTOOLS_DLL_DECL size_t tc_malloc_size(void* ptr) __THROW {
1649 return MallocExtension::instance()->GetAllocatedSize(ptr); 1582 return MallocExtension::instance()->GetAllocatedSize(ptr);
1650 } 1583 }
1651 1584
1652 #endif // TCMALLOC_USING_DEBUGALLOCATION 1585 #endif // TCMALLOC_USING_DEBUGALLOCATION
OLDNEW
« no previous file with comments | « third_party/tcmalloc/vendor/src/system-alloc.cc ('k') | third_party/tcmalloc/vendor/src/tests/current_allocated_bytes_test.cc » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698