| OLD | NEW |
| 1 // Copyright (c) 2007, Google Inc. | 1 // Copyright (c) 2007, Google Inc. |
| 2 // All rights reserved. | 2 // All rights reserved. |
| 3 // | 3 // |
| 4 // Redistribution and use in source and binary forms, with or without | 4 // Redistribution and use in source and binary forms, with or without |
| 5 // modification, are permitted provided that the following conditions are | 5 // modification, are permitted provided that the following conditions are |
| 6 // met: | 6 // met: |
| 7 // | 7 // |
| 8 // * Redistributions of source code must retain the above copyright | 8 // * Redistributions of source code must retain the above copyright |
| 9 // notice, this list of conditions and the following disclaimer. | 9 // notice, this list of conditions and the following disclaimer. |
| 10 // * Redistributions in binary form must reproduce the above | 10 // * Redistributions in binary form must reproduce the above |
| (...skipping 36 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 47 #endif | 47 #endif |
| 48 #include <stdio.h> // for snprintf | 48 #include <stdio.h> // for snprintf |
| 49 #include <stdlib.h> // for mkstemp | 49 #include <stdlib.h> // for mkstemp |
| 50 #include <string.h> // for strerror | 50 #include <string.h> // for strerror |
| 51 #include <sys/mman.h> // for mmap, MAP_FAILED, etc | 51 #include <sys/mman.h> // for mmap, MAP_FAILED, etc |
| 52 #include <sys/statfs.h> // for fstatfs, statfs | 52 #include <sys/statfs.h> // for fstatfs, statfs |
| 53 #include <unistd.h> // for ftruncate, off_t, unlink | 53 #include <unistd.h> // for ftruncate, off_t, unlink |
| 54 #include <new> // for operator new | 54 #include <new> // for operator new |
| 55 #include <string> | 55 #include <string> |
| 56 | 56 |
| 57 #include <gperftools/malloc_extension.h> | 57 #include <google/malloc_extension.h> |
| 58 #include "base/basictypes.h" | 58 #include "base/basictypes.h" |
| 59 #include "base/googleinit.h" | 59 #include "base/googleinit.h" |
| 60 #include "base/sysinfo.h" | 60 #include "base/sysinfo.h" |
| 61 #include "system-alloc.h" |
| 61 #include "internal_logging.h" | 62 #include "internal_logging.h" |
| 62 | 63 |
| 63 // TODO(sanjay): Move the code below into the tcmalloc namespace | |
| 64 using tcmalloc::kLog; | |
| 65 using tcmalloc::kCrash; | |
| 66 using tcmalloc::Log; | |
| 67 using std::string; | 64 using std::string; |
| 68 | 65 |
| 69 DEFINE_string(memfs_malloc_path, EnvToString("TCMALLOC_MEMFS_MALLOC_PATH", ""), | 66 DEFINE_string(memfs_malloc_path, EnvToString("TCMALLOC_MEMFS_MALLOC_PATH", ""), |
| 70 "Path where hugetlbfs or tmpfs is mounted. The caller is " | 67 "Path where hugetlbfs or tmpfs is mounted. The caller is " |
| 71 "responsible for ensuring that the path is unique and does " | 68 "responsible for ensuring that the path is unique and does " |
| 72 "not conflict with another process"); | 69 "not conflict with another process"); |
| 73 DEFINE_int64(memfs_malloc_limit_mb, | 70 DEFINE_int64(memfs_malloc_limit_mb, |
| 74 EnvToInt("TCMALLOC_MEMFS_LIMIT_MB", 0), | 71 EnvToInt("TCMALLOC_MEMFS_LIMIT_MB", 0), |
| 75 "Limit total allocation size to the " | 72 "Limit total allocation size to the " |
| 76 "specified number of MiB. 0 == no limit."); | 73 "specified number of MiB. 0 == no limit."); |
| 77 DEFINE_bool(memfs_malloc_abort_on_fail, | 74 DEFINE_bool(memfs_malloc_abort_on_fail, |
| 78 EnvToBool("TCMALLOC_MEMFS_ABORT_ON_FAIL", false), | 75 EnvToBool("TCMALLOC_MEMFS_ABORT_ON_FAIL", false), |
| 79 "abort() whenever memfs_malloc fails to satisfy an allocation " | 76 "abort() whenever memfs_malloc fails to satisfy an allocation " |
| 80 "for any reason."); | 77 "for any reason."); |
| 81 DEFINE_bool(memfs_malloc_ignore_mmap_fail, | 78 DEFINE_bool(memfs_malloc_ignore_mmap_fail, |
| 82 EnvToBool("TCMALLOC_MEMFS_IGNORE_MMAP_FAIL", false), | 79 EnvToBool("TCMALLOC_MEMFS_IGNORE_MMAP_FAIL", false), |
| 83 "Ignore failures from mmap"); | 80 "Ignore failures from mmap"); |
| 84 DEFINE_bool(memfs_malloc_map_private, | 81 DEFINE_bool(memfs_malloc_map_private, |
| 85 EnvToBool("TCMALLOC_MEMFS_MAP_PRIVATE", false), | 82 EnvToBool("TCMALLOC_MEMFS_MAP_PRIVATE", false), |
| 86 "Use MAP_PRIVATE with mmap"); | 83 "Use MAP_PRIVATE with mmap"); |
| 87 | 84 |
| 88 // Hugetlbfs based allocator for tcmalloc | 85 // Hugetlbfs based allocator for tcmalloc |
| 89 class HugetlbSysAllocator: public SysAllocator { | 86 class HugetlbSysAllocator: public SysAllocator { |
| 90 public: | 87 public: |
| 91 explicit HugetlbSysAllocator(SysAllocator* fallback) | 88 explicit HugetlbSysAllocator(SysAllocator* fallback) |
| 92 : failed_(true), // To disable allocator until Initialize() is called. | 89 : failed_(true), // Unusable until FlagsInitialized() is called |
| 93 big_page_size_(0), | 90 big_page_size_(0), |
| 94 hugetlb_fd_(-1), | 91 hugetlb_fd_(-1), |
| 95 hugetlb_base_(0), | 92 hugetlb_base_(0), |
| 96 fallback_(fallback) { | 93 fallback_(fallback) { |
| 97 } | 94 } |
| 98 | 95 |
| 99 void* Alloc(size_t size, size_t *actual_size, size_t alignment); | 96 void* Alloc(size_t size, size_t *actual_size, size_t alignment); |
| 100 bool Initialize(); | 97 |
| 98 void FlagsInitialized(); |
| 101 | 99 |
| 102 bool failed_; // Whether failed to allocate memory. | 100 bool failed_; // Whether failed to allocate memory. |
| 103 | |
| 104 private: | 101 private: |
| 105 void* AllocInternal(size_t size, size_t *actual_size, size_t alignment); | 102 void* AllocInternal(size_t size, size_t *actual_size, size_t alignment); |
| 106 | 103 |
| 107 int64 big_page_size_; | 104 int64 big_page_size_; |
| 108 int hugetlb_fd_; // file descriptor for hugetlb | 105 int hugetlb_fd_; // file descriptor for hugetlb |
| 109 off_t hugetlb_base_; | 106 off_t hugetlb_base_; |
| 110 | 107 |
| 111 SysAllocator* fallback_; // Default system allocator to fall back to. | 108 SysAllocator* fallback_; // Default system allocator to fall back to. |
| 112 }; | 109 }; |
| 113 static char hugetlb_space[sizeof(HugetlbSysAllocator)]; | 110 static char hugetlb_space[sizeof(HugetlbSysAllocator)]; |
| (...skipping 18 matching lines...) Expand all Loading... |
| 132 size_t aligned_size = ((size + new_alignment - 1) / | 129 size_t aligned_size = ((size + new_alignment - 1) / |
| 133 new_alignment) * new_alignment; | 130 new_alignment) * new_alignment; |
| 134 if (aligned_size < size) { | 131 if (aligned_size < size) { |
| 135 return fallback_->Alloc(size, actual_size, alignment); | 132 return fallback_->Alloc(size, actual_size, alignment); |
| 136 } | 133 } |
| 137 | 134 |
| 138 void* result = AllocInternal(aligned_size, actual_size, new_alignment); | 135 void* result = AllocInternal(aligned_size, actual_size, new_alignment); |
| 139 if (result != NULL) { | 136 if (result != NULL) { |
| 140 return result; | 137 return result; |
| 141 } | 138 } |
| 142 Log(kLog, __FILE__, __LINE__, | 139 TCMalloc_MESSAGE(__FILE__, __LINE__, |
| 143 "HugetlbSysAllocator: (failed, allocated)", failed_, hugetlb_base_); | 140 "HugetlbSysAllocator: failed_=%d allocated=%"PRId64"\n", |
| 141 failed_, static_cast<int64_t>(hugetlb_base_)); |
| 144 if (FLAGS_memfs_malloc_abort_on_fail) { | 142 if (FLAGS_memfs_malloc_abort_on_fail) { |
| 145 Log(kCrash, __FILE__, __LINE__, | 143 CRASH("memfs_malloc_abort_on_fail is set\n"); |
| 146 "memfs_malloc_abort_on_fail is set"); | |
| 147 } | 144 } |
| 148 return fallback_->Alloc(size, actual_size, alignment); | 145 return fallback_->Alloc(size, actual_size, alignment); |
| 149 } | 146 } |
| 150 | 147 |
| 151 void* HugetlbSysAllocator::AllocInternal(size_t size, size_t* actual_size, | 148 void* HugetlbSysAllocator::AllocInternal(size_t size, size_t* actual_size, |
| 152 size_t alignment) { | 149 size_t alignment) { |
| 153 // Ask for extra memory if alignment > pagesize | 150 // Ask for extra memory if alignment > pagesize |
| 154 size_t extra = 0; | 151 size_t extra = 0; |
| 155 if (alignment > big_page_size_) { | 152 if (alignment > big_page_size_) { |
| 156 extra = alignment - big_page_size_; | 153 extra = alignment - big_page_size_; |
| 157 } | 154 } |
| 158 | 155 |
| 159 // Test if this allocation would put us over the limit. | 156 // Test if this allocation would put us over the limit. |
| 160 off_t limit = FLAGS_memfs_malloc_limit_mb*1024*1024; | 157 off_t limit = FLAGS_memfs_malloc_limit_mb*1024*1024; |
| 161 if (limit > 0 && hugetlb_base_ + size + extra > limit) { | 158 if (limit > 0 && hugetlb_base_ + size + extra > limit) { |
| 162 // Disable the allocator when there's less than one page left. | 159 // Disable the allocator when there's less than one page left. |
| 163 if (limit - hugetlb_base_ < big_page_size_) { | 160 if (limit - hugetlb_base_ < big_page_size_) { |
| 164 Log(kLog, __FILE__, __LINE__, "reached memfs_malloc_limit_mb"); | 161 TCMalloc_MESSAGE(__FILE__, __LINE__, "reached memfs_malloc_limit_mb\n"); |
| 165 failed_ = true; | 162 failed_ = true; |
| 166 } | 163 } |
| 167 else { | 164 else { |
| 168 Log(kLog, __FILE__, __LINE__, | 165 TCMalloc_MESSAGE(__FILE__, __LINE__, "alloc size=%"PRIuS |
| 169 "alloc too large (size, bytes left)", size, limit-hugetlb_base_); | 166 " too large while %"PRId64" bytes remain\n", |
| 167 size, static_cast<int64_t>(limit - hugetlb_base_)); |
| 170 } | 168 } |
| 171 return NULL; | 169 return NULL; |
| 172 } | 170 } |
| 173 | 171 |
| 174 // This is not needed for hugetlbfs, but needed for tmpfs. Annoyingly | 172 // This is not needed for hugetlbfs, but needed for tmpfs. Annoyingly |
| 175 // hugetlbfs returns EINVAL for ftruncate. | 173 // hugetlbfs returns EINVAL for ftruncate. |
| 176 int ret = ftruncate(hugetlb_fd_, hugetlb_base_ + size + extra); | 174 int ret = ftruncate(hugetlb_fd_, hugetlb_base_ + size + extra); |
| 177 if (ret != 0 && errno != EINVAL) { | 175 if (ret != 0 && errno != EINVAL) { |
| 178 Log(kLog, __FILE__, __LINE__, | 176 TCMalloc_MESSAGE(__FILE__, __LINE__, "ftruncate failed: %s\n", |
| 179 "ftruncate failed", strerror(errno)); | 177 strerror(errno)); |
| 180 failed_ = true; | 178 failed_ = true; |
| 181 return NULL; | 179 return NULL; |
| 182 } | 180 } |
| 183 | 181 |
| 184 // Note: size + extra does not overflow since: | 182 // Note: size + extra does not overflow since: |
| 185 // size + alignment < (1<<NBITS). | 183 // size + alignment < (1<<NBITS). |
| 186 // and extra <= alignment | 184 // and extra <= alignment |
| 187 // therefore size + extra < (1<<NBITS) | 185 // therefore size + extra < (1<<NBITS) |
| 188 void *result; | 186 void *result; |
| 189 result = mmap(0, size + extra, PROT_WRITE|PROT_READ, | 187 result = mmap(0, size + extra, PROT_WRITE|PROT_READ, |
| 190 FLAGS_memfs_malloc_map_private ? MAP_PRIVATE : MAP_SHARED, | 188 FLAGS_memfs_malloc_map_private ? MAP_PRIVATE : MAP_SHARED, |
| 191 hugetlb_fd_, hugetlb_base_); | 189 hugetlb_fd_, hugetlb_base_); |
| 192 if (result == reinterpret_cast<void*>(MAP_FAILED)) { | 190 if (result == reinterpret_cast<void*>(MAP_FAILED)) { |
| 193 if (!FLAGS_memfs_malloc_ignore_mmap_fail) { | 191 if (!FLAGS_memfs_malloc_ignore_mmap_fail) { |
| 194 Log(kLog, __FILE__, __LINE__, | 192 TCMalloc_MESSAGE(__FILE__, __LINE__, "mmap of size %"PRIuS" failed: %s\n", |
| 195 "mmap failed (size, error)", size + extra, strerror(errno)); | 193 size + extra, strerror(errno)); |
| 196 failed_ = true; | 194 failed_ = true; |
| 197 } | 195 } |
| 198 return NULL; | 196 return NULL; |
| 199 } | 197 } |
| 200 uintptr_t ptr = reinterpret_cast<uintptr_t>(result); | 198 uintptr_t ptr = reinterpret_cast<uintptr_t>(result); |
| 201 | 199 |
| 202 // Adjust the return memory so it is aligned | 200 // Adjust the return memory so it is aligned |
| 203 size_t adjust = 0; | 201 size_t adjust = 0; |
| 204 if ((ptr & (alignment - 1)) != 0) { | 202 if ((ptr & (alignment - 1)) != 0) { |
| 205 adjust = alignment - (ptr & (alignment - 1)); | 203 adjust = alignment - (ptr & (alignment - 1)); |
| 206 } | 204 } |
| 207 ptr += adjust; | 205 ptr += adjust; |
| 208 hugetlb_base_ += (size + extra); | 206 hugetlb_base_ += (size + extra); |
| 209 | 207 |
| 210 if (actual_size) { | 208 if (actual_size) { |
| 211 *actual_size = size + extra - adjust; | 209 *actual_size = size + extra - adjust; |
| 212 } | 210 } |
| 213 | 211 |
| 214 return reinterpret_cast<void*>(ptr); | 212 return reinterpret_cast<void*>(ptr); |
| 215 } | 213 } |
| 216 | 214 |
| 217 bool HugetlbSysAllocator::Initialize() { | 215 void HugetlbSysAllocator::FlagsInitialized() { |
| 218 char path[PATH_MAX]; | 216 if (FLAGS_memfs_malloc_path.length()) { |
| 219 const int pathlen = FLAGS_memfs_malloc_path.size(); | 217 char path[PATH_MAX]; |
| 220 if (pathlen + 8 > sizeof(path)) { | 218 int rc = snprintf(path, sizeof(path), "%s.XXXXXX", |
| 221 Log(kCrash, __FILE__, __LINE__, "XX fatal: memfs_malloc_path too long"); | 219 FLAGS_memfs_malloc_path.c_str()); |
| 222 return false; | 220 if (rc < 0 || rc >= sizeof(path)) { |
| 221 CRASH("XX fatal: memfs_malloc_path too long\n"); |
| 222 } |
| 223 |
| 224 int hugetlb_fd = mkstemp(path); |
| 225 if (hugetlb_fd == -1) { |
| 226 TCMalloc_MESSAGE(__FILE__, __LINE__, |
| 227 "warning: unable to create memfs_malloc_path %s: %s\n", |
| 228 path, strerror(errno)); |
| 229 return; |
| 230 } |
| 231 |
| 232 // Cleanup memory on process exit |
| 233 if (unlink(path) == -1) { |
| 234 CRASH("fatal: error unlinking memfs_malloc_path %s: %s\n", |
| 235 path, strerror(errno)); |
| 236 } |
| 237 |
| 238 // Use fstatfs to figure out the default page size for memfs |
| 239 struct statfs sfs; |
| 240 if (fstatfs(hugetlb_fd, &sfs) == -1) { |
| 241 CRASH("fatal: error fstatfs of memfs_malloc_path: %s\n", |
| 242 strerror(errno)); |
| 243 } |
| 244 int64 page_size = sfs.f_bsize; |
| 245 |
| 246 hugetlb_fd_ = hugetlb_fd; |
| 247 big_page_size_ = page_size; |
| 248 failed_ = false; |
| 223 } | 249 } |
| 224 memcpy(path, FLAGS_memfs_malloc_path.data(), pathlen); | |
| 225 memcpy(path + pathlen, ".XXXXXX", 8); // Also copies terminating \0 | |
| 226 | |
| 227 int hugetlb_fd = mkstemp(path); | |
| 228 if (hugetlb_fd == -1) { | |
| 229 Log(kLog, __FILE__, __LINE__, | |
| 230 "warning: unable to create memfs_malloc_path", | |
| 231 path, strerror(errno)); | |
| 232 return false; | |
| 233 } | |
| 234 | |
| 235 // Cleanup memory on process exit | |
| 236 if (unlink(path) == -1) { | |
| 237 Log(kCrash, __FILE__, __LINE__, | |
| 238 "fatal: error unlinking memfs_malloc_path", path, strerror(errno)); | |
| 239 return false; | |
| 240 } | |
| 241 | |
| 242 // Use fstatfs to figure out the default page size for memfs | |
| 243 struct statfs sfs; | |
| 244 if (fstatfs(hugetlb_fd, &sfs) == -1) { | |
| 245 Log(kCrash, __FILE__, __LINE__, | |
| 246 "fatal: error fstatfs of memfs_malloc_path", strerror(errno)); | |
| 247 return false; | |
| 248 } | |
| 249 int64 page_size = sfs.f_bsize; | |
| 250 | |
| 251 hugetlb_fd_ = hugetlb_fd; | |
| 252 big_page_size_ = page_size; | |
| 253 failed_ = false; | |
| 254 return true; | |
| 255 } | 250 } |
| 256 | 251 |
| 257 REGISTER_MODULE_INITIALIZER(memfs_malloc, { | 252 static void InitSystemAllocator() { |
| 258 if (FLAGS_memfs_malloc_path.length()) { | 253 SysAllocator *alloc = MallocExtension::instance()->GetSystemAllocator(); |
| 259 SysAllocator* alloc = MallocExtension::instance()->GetSystemAllocator(); | 254 HugetlbSysAllocator *hugetlb = new (hugetlb_space) HugetlbSysAllocator(alloc); |
| 260 HugetlbSysAllocator* hp = new (hugetlb_space) HugetlbSysAllocator(alloc); | 255 MallocExtension::instance()->SetSystemAllocator(hugetlb); |
| 261 if (hp->Initialize()) { | 256 } |
| 262 MallocExtension::instance()->SetSystemAllocator(hp); | 257 |
| 263 } | 258 REGISTER_MODULE_INITIALIZER(memfs_malloc, { InitSystemAllocator(); }); |
| 264 } | |
| 265 }); | |
| 266 | 259 |
| 267 #endif /* ifdef __linux */ | 260 #endif /* ifdef __linux */ |
| OLD | NEW |