OLD | NEW |
1 // Copyright (c) 2008, Google Inc. | 1 // Copyright (c) 2008, Google Inc. |
2 // All rights reserved. | 2 // All rights reserved. |
3 // | 3 // |
4 // Redistribution and use in source and binary forms, with or without | 4 // Redistribution and use in source and binary forms, with or without |
5 // modification, are permitted provided that the following conditions are | 5 // modification, are permitted provided that the following conditions are |
6 // met: | 6 // met: |
7 // | 7 // |
8 // * Redistributions of source code must retain the above copyright | 8 // * Redistributions of source code must retain the above copyright |
9 // notice, this list of conditions and the following disclaimer. | 9 // notice, this list of conditions and the following disclaimer. |
10 // * Redistributions in binary form must reproduce the above | 10 // * Redistributions in binary form must reproduce the above |
(...skipping 14 matching lines...) Expand all Loading... |
25 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | 25 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
26 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | 26 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
27 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | 27 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
28 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | 28 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
29 | 29 |
30 // --- | 30 // --- |
31 // Author: Ken Ashcraft <opensource@google.com> | 31 // Author: Ken Ashcraft <opensource@google.com> |
32 | 32 |
33 #include <config.h> | 33 #include <config.h> |
34 #include "thread_cache.h" | 34 #include "thread_cache.h" |
| 35 #include <errno.h> |
35 #include <string.h> // for memcpy | 36 #include <string.h> // for memcpy |
36 #include <algorithm> // for max, min | 37 #include <algorithm> // for max, min |
37 #include "base/commandlineflags.h" // for SpinLockHolder | 38 #include "base/commandlineflags.h" // for SpinLockHolder |
38 #include "base/spinlock.h" // for SpinLockHolder | 39 #include "base/spinlock.h" // for SpinLockHolder |
39 #include "central_freelist.h" // for CentralFreeListPadded | 40 #include "central_freelist.h" // for CentralFreeListPadded |
40 #include "maybe_threads.h" | 41 #include "maybe_threads.h" |
41 | 42 |
42 using std::min; | 43 using std::min; |
43 using std::max; | 44 using std::max; |
44 | 45 |
45 DEFINE_int64(tcmalloc_max_total_thread_cache_bytes, | 46 DEFINE_int64(tcmalloc_max_total_thread_cache_bytes, |
46 EnvToInt64("TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES", | 47 EnvToInt64("TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES", |
47 kDefaultOverallThreadCacheSize), | 48 kDefaultOverallThreadCacheSize), |
48 "Bound on the total amount of bytes allocated to " | 49 "Bound on the total amount of bytes allocated to " |
49 "thread caches. This bound is not strict, so it is possible " | 50 "thread caches. This bound is not strict, so it is possible " |
50 "for the cache to go over this bound in certain circumstances. "); | 51 "for the cache to go over this bound in certain circumstances. " |
| 52 "Maximum value of this flag is capped to 1 GB."); |
51 | 53 |
52 namespace tcmalloc { | 54 namespace tcmalloc { |
53 | 55 |
54 static bool phinited = false; | 56 static bool phinited = false; |
55 | 57 |
56 volatile size_t ThreadCache::per_thread_cache_size_ = kMaxThreadCacheSize; | 58 volatile size_t ThreadCache::per_thread_cache_size_ = kMaxThreadCacheSize; |
57 size_t ThreadCache::overall_thread_cache_size_ = kDefaultOverallThreadCacheSize; | 59 size_t ThreadCache::overall_thread_cache_size_ = kDefaultOverallThreadCacheSize; |
58 ssize_t ThreadCache::unclaimed_cache_space_ = kDefaultOverallThreadCacheSize; | 60 ssize_t ThreadCache::unclaimed_cache_space_ = kDefaultOverallThreadCacheSize; |
59 PageHeapAllocator<ThreadCache> threadcache_allocator; | 61 PageHeapAllocator<ThreadCache> threadcache_allocator; |
60 ThreadCache* ThreadCache::thread_heaps_ = NULL; | 62 ThreadCache* ThreadCache::thread_heaps_ = NULL; |
61 int ThreadCache::thread_heap_count_ = 0; | 63 int ThreadCache::thread_heap_count_ = 0; |
62 ThreadCache* ThreadCache::next_memory_steal_ = NULL; | 64 ThreadCache* ThreadCache::next_memory_steal_ = NULL; |
63 #ifdef HAVE_TLS | 65 #ifdef HAVE_TLS |
64 __thread ThreadCache* ThreadCache::threadlocal_heap_ | 66 __thread ThreadCache* ThreadCache::threadlocal_heap_ |
65 # ifdef HAVE___ATTRIBUTE__ | 67 # ifdef HAVE___ATTRIBUTE__ |
66 __attribute__ ((tls_model ("initial-exec"))) | 68 __attribute__ ((tls_model ("initial-exec"))) |
67 # endif | 69 # endif |
68 ; | 70 ; |
69 #endif | 71 #endif |
70 bool ThreadCache::tsd_inited_ = false; | 72 bool ThreadCache::tsd_inited_ = false; |
71 pthread_key_t ThreadCache::heap_key_; | 73 pthread_key_t ThreadCache::heap_key_; |
72 | 74 |
73 #if defined(HAVE_TLS) | 75 #if defined(HAVE_TLS) |
74 bool kernel_supports_tls = false; // be conservative | 76 bool kernel_supports_tls = false; // be conservative |
75 # if !HAVE_DECL_UNAME // if too old for uname, probably too old for TLS | 77 # if defined(_WIN32) // windows has supported TLS since winnt, I think. |
| 78 void CheckIfKernelSupportsTLS() { |
| 79 kernel_supports_tls = true; |
| 80 } |
| 81 # elif !HAVE_DECL_UNAME // if too old for uname, probably too old for TLS |
76 void CheckIfKernelSupportsTLS() { | 82 void CheckIfKernelSupportsTLS() { |
77 kernel_supports_tls = false; | 83 kernel_supports_tls = false; |
78 } | 84 } |
79 # else | 85 # else |
80 # include <sys/utsname.h> // DECL_UNAME checked for <sys/utsname.h> too | 86 # include <sys/utsname.h> // DECL_UNAME checked for <sys/utsname.h> too |
81 void CheckIfKernelSupportsTLS() { | 87 void CheckIfKernelSupportsTLS() { |
82 struct utsname buf; | 88 struct utsname buf; |
83 if (uname(&buf) != 0) { // should be impossible | 89 if (uname(&buf) < 0) { // should be impossible |
84 MESSAGE("uname failed assuming no TLS support (errno=%d)\n", errno); | 90 Log(kLog, __FILE__, __LINE__, |
| 91 "uname failed assuming no TLS support (errno)", errno); |
85 kernel_supports_tls = false; | 92 kernel_supports_tls = false; |
86 } else if (strcasecmp(buf.sysname, "linux") == 0) { | 93 } else if (strcasecmp(buf.sysname, "linux") == 0) { |
87 // The linux case: the first kernel to support TLS was 2.6.0 | 94 // The linux case: the first kernel to support TLS was 2.6.0 |
88 if (buf.release[0] < '2' && buf.release[1] == '.') // 0.x or 1.x | 95 if (buf.release[0] < '2' && buf.release[1] == '.') // 0.x or 1.x |
89 kernel_supports_tls = false; | 96 kernel_supports_tls = false; |
90 else if (buf.release[0] == '2' && buf.release[1] == '.' && | 97 else if (buf.release[0] == '2' && buf.release[1] == '.' && |
91 buf.release[2] >= '0' && buf.release[2] < '6' && | 98 buf.release[2] >= '0' && buf.release[2] < '6' && |
92 buf.release[3] == '.') // 2.0 - 2.5 | 99 buf.release[3] == '.') // 2.0 - 2.5 |
93 kernel_supports_tls = false; | 100 kernel_supports_tls = false; |
94 else | 101 else |
95 kernel_supports_tls = true; | 102 kernel_supports_tls = true; |
| 103 } else if (strcasecmp(buf.sysname, "CYGWIN_NT-6.1-WOW64") == 0) { |
| 104 // In my testing, this version of cygwin, at least, would hang |
| 105 // when using TLS. |
| 106 kernel_supports_tls = false; |
96 } else { // some other kernel, we'll be optimisitic | 107 } else { // some other kernel, we'll be optimisitic |
97 kernel_supports_tls = true; | 108 kernel_supports_tls = true; |
98 } | 109 } |
99 // TODO(csilvers): VLOG(1) the tls status once we support RAW_VLOG | 110 // TODO(csilvers): VLOG(1) the tls status once we support RAW_VLOG |
100 } | 111 } |
101 # endif // HAVE_DECL_UNAME | 112 # endif // HAVE_DECL_UNAME |
102 #endif // HAVE_TLS | 113 #endif // HAVE_TLS |
103 | 114 |
104 void ThreadCache::Init(pthread_t tid) { | 115 void ThreadCache::Init(pthread_t tid) { |
105 size_ = 0; | 116 size_ = 0; |
(...skipping 146 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
252 const int batch_size = Static::sizemap()->num_objects_to_move(cl); | 263 const int batch_size = Static::sizemap()->num_objects_to_move(cl); |
253 if (list->max_length() > batch_size) { | 264 if (list->max_length() > batch_size) { |
254 list->set_max_length( | 265 list->set_max_length( |
255 max<int>(list->max_length() - batch_size, batch_size)); | 266 max<int>(list->max_length() - batch_size, batch_size)); |
256 } | 267 } |
257 } | 268 } |
258 list->clear_lowwatermark(); | 269 list->clear_lowwatermark(); |
259 } | 270 } |
260 | 271 |
261 IncreaseCacheLimit(); | 272 IncreaseCacheLimit(); |
262 | |
263 // int64 finish = CycleClock::Now(); | |
264 // CycleTimer ct; | |
265 // MESSAGE("GC: %.0f ns\n", ct.CyclesToUsec(finish-start)*1000.0); | |
266 } | 273 } |
267 | 274 |
268 void ThreadCache::IncreaseCacheLimit() { | 275 void ThreadCache::IncreaseCacheLimit() { |
269 SpinLockHolder h(Static::pageheap_lock()); | 276 SpinLockHolder h(Static::pageheap_lock()); |
270 IncreaseCacheLimitLocked(); | 277 IncreaseCacheLimitLocked(); |
271 } | 278 } |
272 | 279 |
273 void ThreadCache::IncreaseCacheLimitLocked() { | 280 void ThreadCache::IncreaseCacheLimitLocked() { |
274 if (unclaimed_cache_space_ > 0) { | 281 if (unclaimed_cache_space_ > 0) { |
275 // Possibly make unclaimed_cache_space_ negative. | 282 // Possibly make unclaimed_cache_space_ negative. |
(...skipping 39 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
315 Static::InitStaticVars(); | 322 Static::InitStaticVars(); |
316 threadcache_allocator.Init(); | 323 threadcache_allocator.Init(); |
317 phinited = 1; | 324 phinited = 1; |
318 } | 325 } |
319 } | 326 } |
320 | 327 |
321 void ThreadCache::InitTSD() { | 328 void ThreadCache::InitTSD() { |
322 ASSERT(!tsd_inited_); | 329 ASSERT(!tsd_inited_); |
323 perftools_pthread_key_create(&heap_key_, DestroyThreadCache); | 330 perftools_pthread_key_create(&heap_key_, DestroyThreadCache); |
324 tsd_inited_ = true; | 331 tsd_inited_ = true; |
| 332 |
| 333 #ifdef PTHREADS_CRASHES_IF_RUN_TOO_EARLY |
| 334 // We may have used a fake pthread_t for the main thread. Fix it. |
| 335 pthread_t zero; |
| 336 memset(&zero, 0, sizeof(zero)); |
| 337 SpinLockHolder h(Static::pageheap_lock()); |
| 338 for (ThreadCache* h = thread_heaps_; h != NULL; h = h->next_) { |
| 339 if (h->tid_ == zero) { |
| 340 h->tid_ = pthread_self(); |
| 341 } |
| 342 } |
| 343 #endif |
325 } | 344 } |
326 | 345 |
327 ThreadCache* ThreadCache::CreateCacheIfNecessary() { | 346 ThreadCache* ThreadCache::CreateCacheIfNecessary() { |
328 // Initialize per-thread data if necessary | 347 // Initialize per-thread data if necessary |
329 ThreadCache* heap = NULL; | 348 ThreadCache* heap = NULL; |
330 { | 349 { |
331 SpinLockHolder h(Static::pageheap_lock()); | 350 SpinLockHolder h(Static::pageheap_lock()); |
332 // On very old libc's, this call may crash if it happens too | 351 // On some old glibc's, and on freebsd's libc (as of freebsd 8.1), |
333 // early. No libc using NPTL should be affected. If there | 352 // calling pthread routines (even pthread_self) too early could |
334 // is a crash here, we could use code (on linux, at least) | 353 // cause a segfault. Since we can call pthreads quite early, we |
335 // to detect NPTL vs LinuxThreads: | 354 // have to protect against that in such situations by making a |
336 // http://www.redhat.com/archives/phil-list/2003-April/msg00038.html | 355 // 'fake' pthread. This is not ideal since it doesn't work well |
337 // If we detect not-NPTL, we could execute the old code from | 356 // when linking tcmalloc statically with apps that create threads |
338 // http://google-perftools.googlecode.com/svn/tags/google-perftools-1.7/sr
c/thread_cache.cc | 357 // before main, so we only do it if we have to. |
339 // that avoids calling pthread_self too early. The problem with | 358 #ifdef PTHREADS_CRASHES_IF_RUN_TOO_EARLY |
340 // that code is it caused a race condition when tcmalloc is linked | 359 pthread_t me; |
341 // in statically and other libraries spawn threads before main. | 360 if (!tsd_inited_) { |
| 361 memset(&me, 0, sizeof(me)); |
| 362 } else { |
| 363 me = pthread_self(); |
| 364 } |
| 365 #else |
342 const pthread_t me = pthread_self(); | 366 const pthread_t me = pthread_self(); |
| 367 #endif |
343 | 368 |
344 // This may be a recursive malloc call from pthread_setspecific() | 369 // This may be a recursive malloc call from pthread_setspecific() |
345 // In that case, the heap for this thread has already been created | 370 // In that case, the heap for this thread has already been created |
346 // and added to the linked list. So we search for that first. | 371 // and added to the linked list. So we search for that first. |
347 for (ThreadCache* h = thread_heaps_; h != NULL; h = h->next_) { | 372 for (ThreadCache* h = thread_heaps_; h != NULL; h = h->next_) { |
348 if (h->tid_ == me) { | 373 if (h->tid_ == me) { |
349 heap = h; | 374 heap = h; |
350 break; | 375 break; |
351 } | 376 } |
352 } | 377 } |
(...skipping 102 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
455 for (ThreadCache* h = thread_heaps_; h != NULL; h = h->next_) { | 480 for (ThreadCache* h = thread_heaps_; h != NULL; h = h->next_) { |
456 // Increasing the total cache size should not circumvent the | 481 // Increasing the total cache size should not circumvent the |
457 // slow-start growth of max_size_. | 482 // slow-start growth of max_size_. |
458 if (ratio < 1.0) { | 483 if (ratio < 1.0) { |
459 h->max_size_ = static_cast<size_t>(h->max_size_ * ratio); | 484 h->max_size_ = static_cast<size_t>(h->max_size_ * ratio); |
460 } | 485 } |
461 claimed += h->max_size_; | 486 claimed += h->max_size_; |
462 } | 487 } |
463 unclaimed_cache_space_ = overall_thread_cache_size_ - claimed; | 488 unclaimed_cache_space_ = overall_thread_cache_size_ - claimed; |
464 per_thread_cache_size_ = space; | 489 per_thread_cache_size_ = space; |
465 // TCMalloc_MESSAGE(__FILE__, __LINE__, "Threads %d => cache size %8d\n", n,
int(space)); | |
466 } | |
467 | |
468 void ThreadCache::Print(TCMalloc_Printer* out) const { | |
469 for (int cl = 0; cl < kNumClasses; ++cl) { | |
470 out->printf(" %5" PRIuS " : %4" PRIuS " len; %4d lo; %4"PRIuS | |
471 " max; %4"PRIuS" overages;\n", | |
472 Static::sizemap()->ByteSizeForClass(cl), | |
473 list_[cl].length(), | |
474 list_[cl].lowwatermark(), | |
475 list_[cl].max_length(), | |
476 list_[cl].length_overages()); | |
477 } | |
478 } | |
479 | |
480 void ThreadCache::PrintThreads(TCMalloc_Printer* out) { | |
481 size_t actual_limit = 0; | |
482 for (ThreadCache* h = thread_heaps_; h != NULL; h = h->next_) { | |
483 h->Print(out); | |
484 actual_limit += h->max_size_; | |
485 } | |
486 out->printf("ThreadCache overall: %"PRIuS ", unclaimed: %"PRIuS | |
487 ", actual: %"PRIuS"\n", | |
488 overall_thread_cache_size_, unclaimed_cache_space_, actual_limit); | |
489 } | 490 } |
490 | 491 |
491 void ThreadCache::GetThreadStats(uint64_t* total_bytes, uint64_t* class_count) { | 492 void ThreadCache::GetThreadStats(uint64_t* total_bytes, uint64_t* class_count) { |
492 for (ThreadCache* h = thread_heaps_; h != NULL; h = h->next_) { | 493 for (ThreadCache* h = thread_heaps_; h != NULL; h = h->next_) { |
493 *total_bytes += h->Size(); | 494 *total_bytes += h->Size(); |
494 if (class_count) { | 495 if (class_count) { |
495 for (int cl = 0; cl < kNumClasses; ++cl) { | 496 for (int cl = 0; cl < kNumClasses; ++cl) { |
496 class_count[cl] += h->freelist_length(cl); | 497 class_count[cl] += h->freelist_length(cl); |
497 } | 498 } |
498 } | 499 } |
499 } | 500 } |
500 } | 501 } |
501 | 502 |
502 void ThreadCache::set_overall_thread_cache_size(size_t new_size) { | 503 void ThreadCache::set_overall_thread_cache_size(size_t new_size) { |
503 // Clip the value to a reasonable range | 504 // Clip the value to a reasonable range |
504 if (new_size < kMinThreadCacheSize) new_size = kMinThreadCacheSize; | 505 if (new_size < kMinThreadCacheSize) new_size = kMinThreadCacheSize; |
505 if (new_size > (1<<30)) new_size = (1<<30); // Limit to 1GB | 506 if (new_size > (1<<30)) new_size = (1<<30); // Limit to 1GB |
506 overall_thread_cache_size_ = new_size; | 507 overall_thread_cache_size_ = new_size; |
507 | 508 |
508 RecomputePerThreadCacheSize(); | 509 RecomputePerThreadCacheSize(); |
509 } | 510 } |
510 | 511 |
511 } // namespace tcmalloc | 512 } // namespace tcmalloc |
OLD | NEW |