| OLD | NEW | 
|---|
| 1 // Copyright (c) 2005, Google Inc. | 1 // Copyright (c) 2005, Google Inc. | 
| 2 // All rights reserved. | 2 // All rights reserved. | 
| 3 // | 3 // | 
| 4 // Redistribution and use in source and binary forms, with or without | 4 // Redistribution and use in source and binary forms, with or without | 
| 5 // modification, are permitted provided that the following conditions are | 5 // modification, are permitted provided that the following conditions are | 
| 6 // met: | 6 // met: | 
| 7 // | 7 // | 
| 8 //     * Redistributions of source code must retain the above copyright | 8 //     * Redistributions of source code must retain the above copyright | 
| 9 // notice, this list of conditions and the following disclaimer. | 9 // notice, this list of conditions and the following disclaimer. | 
| 10 //     * Redistributions in binary form must reproduce the above | 10 //     * Redistributions in binary form must reproduce the above | 
| (...skipping 30 matching lines...) Expand all  Loading... | 
| 41 #else | 41 #else | 
| 42 #include <sys/types.h> | 42 #include <sys/types.h> | 
| 43 #endif | 43 #endif | 
| 44 #ifdef HAVE_MMAP | 44 #ifdef HAVE_MMAP | 
| 45 #include <sys/mman.h>                   // for munmap, mmap, MADV_DONTNEED, etc | 45 #include <sys/mman.h>                   // for munmap, mmap, MADV_DONTNEED, etc | 
| 46 #endif | 46 #endif | 
| 47 #ifdef HAVE_UNISTD_H | 47 #ifdef HAVE_UNISTD_H | 
| 48 #include <unistd.h>                     // for sbrk, getpagesize, off_t | 48 #include <unistd.h>                     // for sbrk, getpagesize, off_t | 
| 49 #endif | 49 #endif | 
| 50 #include <new>                          // for operator new | 50 #include <new>                          // for operator new | 
| 51 #include <google/malloc_extension.h> | 51 #include <gperftools/malloc_extension.h> | 
| 52 #include "base/basictypes.h" | 52 #include "base/basictypes.h" | 
| 53 #include "base/commandlineflags.h" | 53 #include "base/commandlineflags.h" | 
| 54 #include "base/spinlock.h"              // for SpinLockHolder, SpinLock, etc | 54 #include "base/spinlock.h"              // for SpinLockHolder, SpinLock, etc | 
| 55 #include "common.h" | 55 #include "common.h" | 
| 56 #include "internal_logging.h" | 56 #include "internal_logging.h" | 
| 57 | 57 | 
| 58 // On systems (like freebsd) that don't define MAP_ANONYMOUS, use the old | 58 // On systems (like freebsd) that don't define MAP_ANONYMOUS, use the old | 
| 59 // form of the name instead. | 59 // form of the name instead. | 
| 60 #ifndef MAP_ANONYMOUS | 60 #ifndef MAP_ANONYMOUS | 
| 61 # define MAP_ANONYMOUS MAP_ANON | 61 # define MAP_ANONYMOUS MAP_ANON | 
| 62 #endif | 62 #endif | 
| 63 | 63 | 
|  | 64 // MADV_FREE is specifically designed for use by malloc(), but only | 
|  | 65 // FreeBSD supports it; in linux we fall back to the somewhat inferior | 
|  | 66 // MADV_DONTNEED. | 
|  | 67 #if !defined(MADV_FREE) && defined(MADV_DONTNEED) | 
|  | 68 # define MADV_FREE  MADV_DONTNEED | 
|  | 69 #endif | 
|  | 70 | 
| 64 // Solaris has a bug where it doesn't declare madvise() for C++. | 71 // Solaris has a bug where it doesn't declare madvise() for C++. | 
| 65 //    http://www.opensolaris.org/jive/thread.jspa?threadID=21035&tstart=0 | 72 //    http://www.opensolaris.org/jive/thread.jspa?threadID=21035&tstart=0 | 
| 66 #if defined(__sun) && defined(__SVR4) | 73 #if defined(__sun) && defined(__SVR4) | 
| 67 # include <sys/types.h>    // for caddr_t | 74 # include <sys/types.h>    // for caddr_t | 
| 68   extern "C" { extern int madvise(caddr_t, size_t, int); } | 75   extern "C" { extern int madvise(caddr_t, size_t, int); } | 
| 69 #endif | 76 #endif | 
| 70 | 77 | 
| 71 // Set kDebugMode mode so that we can have use C++ conditionals | 78 // Set kDebugMode mode so that we can have use C++ conditionals | 
| 72 // instead of preprocessor conditionals. | 79 // instead of preprocessor conditionals. | 
| 73 #ifdef NDEBUG | 80 #ifdef NDEBUG | 
| 74 static const bool kDebugMode = false; | 81 static const bool kDebugMode = false; | 
| 75 #else | 82 #else | 
| 76 static const bool kDebugMode = true; | 83 static const bool kDebugMode = true; | 
| 77 #endif | 84 #endif | 
| 78 | 85 | 
|  | 86 // TODO(sanjay): Move the code below into the tcmalloc namespace | 
|  | 87 using tcmalloc::kLog; | 
|  | 88 using tcmalloc::Log; | 
|  | 89 | 
| 79 // Anonymous namespace to avoid name conflicts on "CheckAddressBits". | 90 // Anonymous namespace to avoid name conflicts on "CheckAddressBits". | 
| 80 namespace { | 91 namespace { | 
| 81 | 92 | 
| 82 // Check that no bit is set at position ADDRESS_BITS or higher. | 93 // Check that no bit is set at position ADDRESS_BITS or higher. | 
| 83 template <int ADDRESS_BITS> bool CheckAddressBits(uintptr_t ptr) { | 94 template <int ADDRESS_BITS> bool CheckAddressBits(uintptr_t ptr) { | 
| 84   return (ptr >> ADDRESS_BITS) == 0; | 95   return (ptr >> ADDRESS_BITS) == 0; | 
| 85 } | 96 } | 
| 86 | 97 | 
| 87 // Specialize for the bit width of a pointer to avoid undefined shift. | 98 // Specialize for the bit width of a pointer to avoid undefined shift. | 
| 88 template <> bool CheckAddressBits<8 * sizeof(void*)>(uintptr_t ptr) { | 99 template <> bool CheckAddressBits<8 * sizeof(void*)>(uintptr_t ptr) { | 
| 89   return true; | 100   return true; | 
| 90 } | 101 } | 
| 91 | 102 | 
| 92 }  // Anonymous namespace to avoid name conflicts on "CheckAddressBits". | 103 }  // Anonymous namespace to avoid name conflicts on "CheckAddressBits". | 
| 93 | 104 | 
| 94 COMPILE_ASSERT(kAddressBits <= 8 * sizeof(void*), | 105 COMPILE_ASSERT(kAddressBits <= 8 * sizeof(void*), | 
| 95                address_bits_larger_than_pointer_size); | 106                address_bits_larger_than_pointer_size); | 
| 96 | 107 | 
| 97 // Structure for discovering alignment | 108 // Structure for discovering alignment | 
| 98 union MemoryAligner { | 109 union MemoryAligner { | 
| 99   void*  p; | 110   void*  p; | 
| 100   double d; | 111   double d; | 
| 101   size_t s; | 112   size_t s; | 
| 102 } CACHELINE_ALIGNED; | 113 } CACHELINE_ALIGNED; | 
| 103 | 114 | 
| 104 static SpinLock spinlock(SpinLock::LINKER_INITIALIZED); | 115 static SpinLock spinlock(SpinLock::LINKER_INITIALIZED); | 
| 105 | 116 | 
|  | 117 #if defined(HAVE_MMAP) || defined(MADV_FREE) | 
| 106 #ifdef HAVE_GETPAGESIZE | 118 #ifdef HAVE_GETPAGESIZE | 
| 107 static size_t pagesize = 0; | 119 static size_t pagesize = 0; | 
| 108 #endif | 120 #endif | 
|  | 121 #endif | 
| 109 | 122 | 
| 110 // The current system allocator | 123 // The current system allocator | 
| 111 SysAllocator* sys_alloc = NULL; | 124 SysAllocator* sys_alloc = NULL; | 
| 112 | 125 | 
| 113 // Configuration parameters. | 126 // Configuration parameters. | 
| 114 DEFINE_int32(malloc_devmem_start, | 127 DEFINE_int32(malloc_devmem_start, | 
| 115              EnvToInt("TCMALLOC_DEVMEM_START", 0), | 128              EnvToInt("TCMALLOC_DEVMEM_START", 0), | 
| 116              "Physical memory starting location in MB for /dev/mem allocation." | 129              "Physical memory starting location in MB for /dev/mem allocation." | 
| 117              "  Setting this to 0 disables /dev/mem allocation"); | 130              "  Setting this to 0 disables /dev/mem allocation"); | 
| 118 DEFINE_int32(malloc_devmem_limit, | 131 DEFINE_int32(malloc_devmem_limit, | 
| 119              EnvToInt("TCMALLOC_DEVMEM_LIMIT", 0), | 132              EnvToInt("TCMALLOC_DEVMEM_LIMIT", 0), | 
| 120              "Physical memory limit location in MB for /dev/mem allocation." | 133              "Physical memory limit location in MB for /dev/mem allocation." | 
| 121              "  Setting this to 0 means no limit."); | 134              "  Setting this to 0 means no limit."); | 
| 122 DEFINE_bool(malloc_skip_sbrk, | 135 DEFINE_bool(malloc_skip_sbrk, | 
| 123             EnvToBool("TCMALLOC_SKIP_SBRK", false), | 136             EnvToBool("TCMALLOC_SKIP_SBRK", false), | 
| 124             "Whether sbrk can be used to obtain memory."); | 137             "Whether sbrk can be used to obtain memory."); | 
| 125 DEFINE_bool(malloc_skip_mmap, | 138 DEFINE_bool(malloc_skip_mmap, | 
| 126             EnvToBool("TCMALLOC_SKIP_MMAP", false), | 139             EnvToBool("TCMALLOC_SKIP_MMAP", false), | 
| 127             "Whether mmap can be used to obtain memory."); | 140             "Whether mmap can be used to obtain memory."); | 
| 128 | 141 | 
| 129 // static allocators | 142 // static allocators | 
| 130 class SbrkSysAllocator : public SysAllocator { | 143 class SbrkSysAllocator : public SysAllocator { | 
| 131 public: | 144 public: | 
| 132   SbrkSysAllocator() : SysAllocator() { | 145   SbrkSysAllocator() : SysAllocator() { | 
| 133   } | 146   } | 
| 134   void* Alloc(size_t size, size_t *actual_size, size_t alignment); | 147   void* Alloc(size_t size, size_t *actual_size, size_t alignment); | 
| 135   void FlagsInitialized() {} |  | 
| 136 }; | 148 }; | 
| 137 static char sbrk_space[sizeof(SbrkSysAllocator)]; | 149 static char sbrk_space[sizeof(SbrkSysAllocator)]; | 
| 138 | 150 | 
| 139 class MmapSysAllocator : public SysAllocator { | 151 class MmapSysAllocator : public SysAllocator { | 
| 140 public: | 152 public: | 
| 141   MmapSysAllocator() : SysAllocator() { | 153   MmapSysAllocator() : SysAllocator() { | 
| 142   } | 154   } | 
| 143   void* Alloc(size_t size, size_t *actual_size, size_t alignment); | 155   void* Alloc(size_t size, size_t *actual_size, size_t alignment); | 
| 144   void FlagsInitialized() {} |  | 
| 145 }; | 156 }; | 
| 146 static char mmap_space[sizeof(MmapSysAllocator)]; | 157 static char mmap_space[sizeof(MmapSysAllocator)]; | 
| 147 | 158 | 
| 148 class DevMemSysAllocator : public SysAllocator { | 159 class DevMemSysAllocator : public SysAllocator { | 
| 149 public: | 160 public: | 
| 150   DevMemSysAllocator() : SysAllocator() { | 161   DevMemSysAllocator() : SysAllocator() { | 
| 151   } | 162   } | 
| 152   void* Alloc(size_t size, size_t *actual_size, size_t alignment); | 163   void* Alloc(size_t size, size_t *actual_size, size_t alignment); | 
| 153   void FlagsInitialized() {} |  | 
| 154 }; | 164 }; | 
| 155 | 165 | 
| 156 class DefaultSysAllocator : public SysAllocator { | 166 class DefaultSysAllocator : public SysAllocator { | 
| 157  public: | 167  public: | 
| 158   DefaultSysAllocator() : SysAllocator() { | 168   DefaultSysAllocator() : SysAllocator() { | 
| 159     for (int i = 0; i < kMaxAllocators; i++) { | 169     for (int i = 0; i < kMaxAllocators; i++) { | 
| 160       failed_[i] = true; | 170       failed_[i] = true; | 
| 161       allocs_[i] = NULL; | 171       allocs_[i] = NULL; | 
| 162       names_[i] = NULL; | 172       names_[i] = NULL; | 
| 163     } | 173     } | 
| 164   } | 174   } | 
| 165   void SetChildAllocator(SysAllocator* alloc, unsigned int index, | 175   void SetChildAllocator(SysAllocator* alloc, unsigned int index, | 
| 166                          const char* name) { | 176                          const char* name) { | 
| 167     if (index < kMaxAllocators && alloc != NULL) { | 177     if (index < kMaxAllocators && alloc != NULL) { | 
| 168       allocs_[index] = alloc; | 178       allocs_[index] = alloc; | 
| 169       failed_[index] = false; | 179       failed_[index] = false; | 
| 170       names_[index] = name; | 180       names_[index] = name; | 
| 171     } | 181     } | 
| 172   } | 182   } | 
| 173   void* Alloc(size_t size, size_t *actual_size, size_t alignment); | 183   void* Alloc(size_t size, size_t *actual_size, size_t alignment); | 
| 174   void FlagsInitialized() {} |  | 
| 175 | 184 | 
| 176  private: | 185  private: | 
| 177   static const int kMaxAllocators = 2; | 186   static const int kMaxAllocators = 2; | 
| 178   bool failed_[kMaxAllocators]; | 187   bool failed_[kMaxAllocators]; | 
| 179   SysAllocator* allocs_[kMaxAllocators]; | 188   SysAllocator* allocs_[kMaxAllocators]; | 
| 180   const char* names_[kMaxAllocators]; | 189   const char* names_[kMaxAllocators]; | 
| 181 }; | 190 }; | 
| 182 static char default_space[sizeof(DefaultSysAllocator)]; | 191 static char default_space[sizeof(DefaultSysAllocator)]; | 
| 183 static const char sbrk_name[] = "SbrkSysAllocator"; | 192 static const char sbrk_name[] = "SbrkSysAllocator"; | 
| 184 static const char mmap_name[] = "MmapSysAllocator"; | 193 static const char mmap_name[] = "MmapSysAllocator"; | 
| (...skipping 228 matching lines...) Expand 10 before | Expand all | Expand 10 after  Loading... | 
| 413 } | 422 } | 
| 414 | 423 | 
| 415 void* DefaultSysAllocator::Alloc(size_t size, size_t *actual_size, | 424 void* DefaultSysAllocator::Alloc(size_t size, size_t *actual_size, | 
| 416                                  size_t alignment) { | 425                                  size_t alignment) { | 
| 417   for (int i = 0; i < kMaxAllocators; i++) { | 426   for (int i = 0; i < kMaxAllocators; i++) { | 
| 418     if (!failed_[i] && allocs_[i] != NULL) { | 427     if (!failed_[i] && allocs_[i] != NULL) { | 
| 419       void* result = allocs_[i]->Alloc(size, actual_size, alignment); | 428       void* result = allocs_[i]->Alloc(size, actual_size, alignment); | 
| 420       if (result != NULL) { | 429       if (result != NULL) { | 
| 421         return result; | 430         return result; | 
| 422       } | 431       } | 
| 423       TCMalloc_MESSAGE(__FILE__, __LINE__, "%s failed.\n", names_[i]); |  | 
| 424       failed_[i] = true; | 432       failed_[i] = true; | 
| 425     } | 433     } | 
| 426   } | 434   } | 
| 427   // After both failed, reset "failed_" to false so that a single failed | 435   // After both failed, reset "failed_" to false so that a single failed | 
| 428   // allocation won't make the allocator never work again. | 436   // allocation won't make the allocator never work again. | 
| 429   for (int i = 0; i < kMaxAllocators; i++) { | 437   for (int i = 0; i < kMaxAllocators; i++) { | 
| 430     failed_[i] = false; | 438     failed_[i] = false; | 
| 431   } | 439   } | 
| 432   return NULL; | 440   return NULL; | 
| 433 } | 441 } | 
| (...skipping 58 matching lines...) Expand 10 before | Expand all | Expand 10 after  Loading... | 
| 492     return 0; | 500     return 0; | 
| 493 | 501 | 
| 494   if (!mprotect(start, pagesize, PROT_NONE)) | 502   if (!mprotect(start, pagesize, PROT_NONE)) | 
| 495     return pagesize; | 503     return pagesize; | 
| 496 #endif | 504 #endif | 
| 497 | 505 | 
| 498   return 0; | 506   return 0; | 
| 499 } | 507 } | 
| 500 | 508 | 
| 501 void TCMalloc_SystemRelease(void* start, size_t length) { | 509 void TCMalloc_SystemRelease(void* start, size_t length) { | 
| 502 #ifdef MADV_DONTNEED | 510 #ifdef MADV_FREE | 
| 503   if (FLAGS_malloc_devmem_start) { | 511   if (FLAGS_malloc_devmem_start) { | 
| 504     // It's not safe to use MADV_DONTNEED if we've been mapping | 512     // It's not safe to use MADV_FREE/MADV_DONTNEED if we've been | 
| 505     // /dev/mem for heap memory | 513     // mapping /dev/mem for heap memory. | 
| 506     return; | 514     return; | 
| 507   } | 515   } | 
| 508   if (pagesize == 0) pagesize = getpagesize(); | 516   if (pagesize == 0) pagesize = getpagesize(); | 
| 509   const size_t pagemask = pagesize - 1; | 517   const size_t pagemask = pagesize - 1; | 
| 510 | 518 | 
| 511   size_t new_start = reinterpret_cast<size_t>(start); | 519   size_t new_start = reinterpret_cast<size_t>(start); | 
| 512   size_t end = new_start + length; | 520   size_t end = new_start + length; | 
| 513   size_t new_end = end; | 521   size_t new_end = end; | 
| 514 | 522 | 
| 515   // Round up the starting address and round down the ending address | 523   // Round up the starting address and round down the ending address | 
| 516   // to be page aligned: | 524   // to be page aligned: | 
| 517   new_start = (new_start + pagesize - 1) & ~pagemask; | 525   new_start = (new_start + pagesize - 1) & ~pagemask; | 
| 518   new_end = new_end & ~pagemask; | 526   new_end = new_end & ~pagemask; | 
| 519 | 527 | 
| 520   ASSERT((new_start & pagemask) == 0); | 528   ASSERT((new_start & pagemask) == 0); | 
| 521   ASSERT((new_end & pagemask) == 0); | 529   ASSERT((new_end & pagemask) == 0); | 
| 522   ASSERT(new_start >= reinterpret_cast<size_t>(start)); | 530   ASSERT(new_start >= reinterpret_cast<size_t>(start)); | 
| 523   ASSERT(new_end <= end); | 531   ASSERT(new_end <= end); | 
| 524 | 532 | 
| 525   if (new_end > new_start) { | 533   if (new_end > new_start) { | 
| 526     // Note -- ignoring most return codes, because if this fails it | 534     // Note -- ignoring most return codes, because if this fails it | 
| 527     // doesn't matter... | 535     // doesn't matter... | 
| 528     while (madvise(reinterpret_cast<char*>(new_start), new_end - new_start, | 536     while (madvise(reinterpret_cast<char*>(new_start), new_end - new_start, | 
| 529                    MADV_DONTNEED) == -1 && | 537                    MADV_FREE) == -1 && | 
| 530            errno == EAGAIN) { | 538            errno == EAGAIN) { | 
| 531       // NOP | 539       // NOP | 
| 532     } | 540     } | 
| 533   } | 541   } | 
| 534 #endif | 542 #endif | 
| 535 } | 543 } | 
| 536 | 544 | 
| 537 void TCMalloc_SystemCommit(void* start, size_t length) { | 545 void TCMalloc_SystemCommit(void* start, size_t length) { | 
| 538   // Nothing to do here.  TCMalloc_SystemRelease does not alter pages | 546   // Nothing to do here.  TCMalloc_SystemRelease does not alter pages | 
| 539   // such that they need to be re-committed before they can be used by the | 547   // such that they need to be re-committed before they can be used by the | 
| 540   // application. | 548   // application. | 
| 541 } | 549 } | 
| OLD | NEW | 
|---|