OLD | NEW |
1 // Copyright (c) 2007, Google Inc. | 1 // Copyright (c) 2007, Google Inc. |
2 // All rights reserved. | 2 // All rights reserved. |
3 // | 3 // |
4 // Redistribution and use in source and binary forms, with or without | 4 // Redistribution and use in source and binary forms, with or without |
5 // modification, are permitted provided that the following conditions are | 5 // modification, are permitted provided that the following conditions are |
6 // met: | 6 // met: |
7 // | 7 // |
8 // * Redistributions of source code must retain the above copyright | 8 // * Redistributions of source code must retain the above copyright |
9 // notice, this list of conditions and the following disclaimer. | 9 // notice, this list of conditions and the following disclaimer. |
10 // * Redistributions in binary form must reproduce the above | 10 // * Redistributions in binary form must reproduce the above |
(...skipping 36 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
47 #endif | 47 #endif |
48 #include <stdio.h> // for snprintf | 48 #include <stdio.h> // for snprintf |
49 #include <stdlib.h> // for mkstemp | 49 #include <stdlib.h> // for mkstemp |
50 #include <string.h> // for strerror | 50 #include <string.h> // for strerror |
51 #include <sys/mman.h> // for mmap, MAP_FAILED, etc | 51 #include <sys/mman.h> // for mmap, MAP_FAILED, etc |
52 #include <sys/statfs.h> // for fstatfs, statfs | 52 #include <sys/statfs.h> // for fstatfs, statfs |
53 #include <unistd.h> // for ftruncate, off_t, unlink | 53 #include <unistd.h> // for ftruncate, off_t, unlink |
54 #include <new> // for operator new | 54 #include <new> // for operator new |
55 #include <string> | 55 #include <string> |
56 | 56 |
57 #include <google/malloc_extension.h> | 57 #include <gperftools/malloc_extension.h> |
58 #include "base/basictypes.h" | 58 #include "base/basictypes.h" |
59 #include "base/googleinit.h" | 59 #include "base/googleinit.h" |
60 #include "base/sysinfo.h" | 60 #include "base/sysinfo.h" |
61 #include "internal_logging.h" | 61 #include "internal_logging.h" |
62 | 62 |
| 63 // TODO(sanjay): Move the code below into the tcmalloc namespace |
| 64 using tcmalloc::kLog; |
| 65 using tcmalloc::kCrash; |
| 66 using tcmalloc::Log; |
63 using std::string; | 67 using std::string; |
64 | 68 |
65 DEFINE_string(memfs_malloc_path, EnvToString("TCMALLOC_MEMFS_MALLOC_PATH", ""), | 69 DEFINE_string(memfs_malloc_path, EnvToString("TCMALLOC_MEMFS_MALLOC_PATH", ""), |
66 "Path where hugetlbfs or tmpfs is mounted. The caller is " | 70 "Path where hugetlbfs or tmpfs is mounted. The caller is " |
67 "responsible for ensuring that the path is unique and does " | 71 "responsible for ensuring that the path is unique and does " |
68 "not conflict with another process"); | 72 "not conflict with another process"); |
69 DEFINE_int64(memfs_malloc_limit_mb, | 73 DEFINE_int64(memfs_malloc_limit_mb, |
70 EnvToInt("TCMALLOC_MEMFS_LIMIT_MB", 0), | 74 EnvToInt("TCMALLOC_MEMFS_LIMIT_MB", 0), |
71 "Limit total allocation size to the " | 75 "Limit total allocation size to the " |
72 "specified number of MiB. 0 == no limit."); | 76 "specified number of MiB. 0 == no limit."); |
(...skipping 55 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
128 size_t aligned_size = ((size + new_alignment - 1) / | 132 size_t aligned_size = ((size + new_alignment - 1) / |
129 new_alignment) * new_alignment; | 133 new_alignment) * new_alignment; |
130 if (aligned_size < size) { | 134 if (aligned_size < size) { |
131 return fallback_->Alloc(size, actual_size, alignment); | 135 return fallback_->Alloc(size, actual_size, alignment); |
132 } | 136 } |
133 | 137 |
134 void* result = AllocInternal(aligned_size, actual_size, new_alignment); | 138 void* result = AllocInternal(aligned_size, actual_size, new_alignment); |
135 if (result != NULL) { | 139 if (result != NULL) { |
136 return result; | 140 return result; |
137 } | 141 } |
138 TCMalloc_MESSAGE(__FILE__, __LINE__, | 142 Log(kLog, __FILE__, __LINE__, |
139 "HugetlbSysAllocator: failed_=%d allocated=%"PRId64"\n", | 143 "HugetlbSysAllocator: (failed, allocated)", failed_, hugetlb_base_); |
140 failed_, static_cast<int64_t>(hugetlb_base_)); | |
141 if (FLAGS_memfs_malloc_abort_on_fail) { | 144 if (FLAGS_memfs_malloc_abort_on_fail) { |
142 CRASH("memfs_malloc_abort_on_fail is set\n"); | 145 Log(kCrash, __FILE__, __LINE__, |
| 146 "memfs_malloc_abort_on_fail is set"); |
143 } | 147 } |
144 return fallback_->Alloc(size, actual_size, alignment); | 148 return fallback_->Alloc(size, actual_size, alignment); |
145 } | 149 } |
146 | 150 |
147 void* HugetlbSysAllocator::AllocInternal(size_t size, size_t* actual_size, | 151 void* HugetlbSysAllocator::AllocInternal(size_t size, size_t* actual_size, |
148 size_t alignment) { | 152 size_t alignment) { |
149 // Ask for extra memory if alignment > pagesize | 153 // Ask for extra memory if alignment > pagesize |
150 size_t extra = 0; | 154 size_t extra = 0; |
151 if (alignment > big_page_size_) { | 155 if (alignment > big_page_size_) { |
152 extra = alignment - big_page_size_; | 156 extra = alignment - big_page_size_; |
153 } | 157 } |
154 | 158 |
155 // Test if this allocation would put us over the limit. | 159 // Test if this allocation would put us over the limit. |
156 off_t limit = FLAGS_memfs_malloc_limit_mb*1024*1024; | 160 off_t limit = FLAGS_memfs_malloc_limit_mb*1024*1024; |
157 if (limit > 0 && hugetlb_base_ + size + extra > limit) { | 161 if (limit > 0 && hugetlb_base_ + size + extra > limit) { |
158 // Disable the allocator when there's less than one page left. | 162 // Disable the allocator when there's less than one page left. |
159 if (limit - hugetlb_base_ < big_page_size_) { | 163 if (limit - hugetlb_base_ < big_page_size_) { |
160 TCMalloc_MESSAGE(__FILE__, __LINE__, "reached memfs_malloc_limit_mb\n"); | 164 Log(kLog, __FILE__, __LINE__, "reached memfs_malloc_limit_mb"); |
161 failed_ = true; | 165 failed_ = true; |
162 } | 166 } |
163 else { | 167 else { |
164 TCMalloc_MESSAGE(__FILE__, __LINE__, "alloc size=%"PRIuS | 168 Log(kLog, __FILE__, __LINE__, |
165 " too large while %"PRId64" bytes remain\n", | 169 "alloc too large (size, bytes left)", size, limit-hugetlb_base_); |
166 size, static_cast<int64_t>(limit - hugetlb_base_)); | |
167 } | 170 } |
168 return NULL; | 171 return NULL; |
169 } | 172 } |
170 | 173 |
171 // This is not needed for hugetlbfs, but needed for tmpfs. Annoyingly | 174 // This is not needed for hugetlbfs, but needed for tmpfs. Annoyingly |
172 // hugetlbfs returns EINVAL for ftruncate. | 175 // hugetlbfs returns EINVAL for ftruncate. |
173 int ret = ftruncate(hugetlb_fd_, hugetlb_base_ + size + extra); | 176 int ret = ftruncate(hugetlb_fd_, hugetlb_base_ + size + extra); |
174 if (ret != 0 && errno != EINVAL) { | 177 if (ret != 0 && errno != EINVAL) { |
175 TCMalloc_MESSAGE(__FILE__, __LINE__, "ftruncate failed: %s\n", | 178 Log(kLog, __FILE__, __LINE__, |
176 strerror(errno)); | 179 "ftruncate failed", strerror(errno)); |
177 failed_ = true; | 180 failed_ = true; |
178 return NULL; | 181 return NULL; |
179 } | 182 } |
180 | 183 |
181 // Note: size + extra does not overflow since: | 184 // Note: size + extra does not overflow since: |
182 // size + alignment < (1<<NBITS). | 185 // size + alignment < (1<<NBITS). |
183 // and extra <= alignment | 186 // and extra <= alignment |
184 // therefore size + extra < (1<<NBITS) | 187 // therefore size + extra < (1<<NBITS) |
185 void *result; | 188 void *result; |
186 result = mmap(0, size + extra, PROT_WRITE|PROT_READ, | 189 result = mmap(0, size + extra, PROT_WRITE|PROT_READ, |
187 FLAGS_memfs_malloc_map_private ? MAP_PRIVATE : MAP_SHARED, | 190 FLAGS_memfs_malloc_map_private ? MAP_PRIVATE : MAP_SHARED, |
188 hugetlb_fd_, hugetlb_base_); | 191 hugetlb_fd_, hugetlb_base_); |
189 if (result == reinterpret_cast<void*>(MAP_FAILED)) { | 192 if (result == reinterpret_cast<void*>(MAP_FAILED)) { |
190 if (!FLAGS_memfs_malloc_ignore_mmap_fail) { | 193 if (!FLAGS_memfs_malloc_ignore_mmap_fail) { |
191 TCMalloc_MESSAGE(__FILE__, __LINE__, "mmap of size %"PRIuS" failed: %s\n", | 194 Log(kLog, __FILE__, __LINE__, |
192 size + extra, strerror(errno)); | 195 "mmap failed (size, error)", size + extra, strerror(errno)); |
193 failed_ = true; | 196 failed_ = true; |
194 } | 197 } |
195 return NULL; | 198 return NULL; |
196 } | 199 } |
197 uintptr_t ptr = reinterpret_cast<uintptr_t>(result); | 200 uintptr_t ptr = reinterpret_cast<uintptr_t>(result); |
198 | 201 |
199 // Adjust the return memory so it is aligned | 202 // Adjust the return memory so it is aligned |
200 size_t adjust = 0; | 203 size_t adjust = 0; |
201 if ((ptr & (alignment - 1)) != 0) { | 204 if ((ptr & (alignment - 1)) != 0) { |
202 adjust = alignment - (ptr & (alignment - 1)); | 205 adjust = alignment - (ptr & (alignment - 1)); |
203 } | 206 } |
204 ptr += adjust; | 207 ptr += adjust; |
205 hugetlb_base_ += (size + extra); | 208 hugetlb_base_ += (size + extra); |
206 | 209 |
207 if (actual_size) { | 210 if (actual_size) { |
208 *actual_size = size + extra - adjust; | 211 *actual_size = size + extra - adjust; |
209 } | 212 } |
210 | 213 |
211 return reinterpret_cast<void*>(ptr); | 214 return reinterpret_cast<void*>(ptr); |
212 } | 215 } |
213 | 216 |
214 bool HugetlbSysAllocator::Initialize() { | 217 bool HugetlbSysAllocator::Initialize() { |
215 char path[PATH_MAX]; | 218 char path[PATH_MAX]; |
216 int rc = snprintf(path, sizeof(path), "%s.XXXXXX", | 219 const int pathlen = FLAGS_memfs_malloc_path.size(); |
217 FLAGS_memfs_malloc_path.c_str()); | 220 if (pathlen + 8 > sizeof(path)) { |
218 if (rc < 0 || rc >= sizeof(path)) { | 221 Log(kCrash, __FILE__, __LINE__, "XX fatal: memfs_malloc_path too long"); |
219 CRASH("XX fatal: memfs_malloc_path too long\n"); | |
220 return false; | 222 return false; |
221 } | 223 } |
| 224 memcpy(path, FLAGS_memfs_malloc_path.data(), pathlen); |
| 225 memcpy(path + pathlen, ".XXXXXX", 8); // Also copies terminating \0 |
222 | 226 |
223 int hugetlb_fd = mkstemp(path); | 227 int hugetlb_fd = mkstemp(path); |
224 if (hugetlb_fd == -1) { | 228 if (hugetlb_fd == -1) { |
225 TCMalloc_MESSAGE(__FILE__, __LINE__, | 229 Log(kLog, __FILE__, __LINE__, |
226 "warning: unable to create memfs_malloc_path %s: %s\n", | 230 "warning: unable to create memfs_malloc_path", |
227 path, strerror(errno)); | 231 path, strerror(errno)); |
228 return false; | 232 return false; |
229 } | 233 } |
230 | 234 |
231 // Cleanup memory on process exit | 235 // Cleanup memory on process exit |
232 if (unlink(path) == -1) { | 236 if (unlink(path) == -1) { |
233 CRASH("fatal: error unlinking memfs_malloc_path %s: %s\n", | 237 Log(kCrash, __FILE__, __LINE__, |
234 path, strerror(errno)); | 238 "fatal: error unlinking memfs_malloc_path", path, strerror(errno)); |
235 return false; | 239 return false; |
236 } | 240 } |
237 | 241 |
238 // Use fstatfs to figure out the default page size for memfs | 242 // Use fstatfs to figure out the default page size for memfs |
239 struct statfs sfs; | 243 struct statfs sfs; |
240 if (fstatfs(hugetlb_fd, &sfs) == -1) { | 244 if (fstatfs(hugetlb_fd, &sfs) == -1) { |
241 CRASH("fatal: error fstatfs of memfs_malloc_path: %s\n", | 245 Log(kCrash, __FILE__, __LINE__, |
242 strerror(errno)); | 246 "fatal: error fstatfs of memfs_malloc_path", strerror(errno)); |
243 return false; | 247 return false; |
244 } | 248 } |
245 int64 page_size = sfs.f_bsize; | 249 int64 page_size = sfs.f_bsize; |
246 | 250 |
247 hugetlb_fd_ = hugetlb_fd; | 251 hugetlb_fd_ = hugetlb_fd; |
248 big_page_size_ = page_size; | 252 big_page_size_ = page_size; |
249 failed_ = false; | 253 failed_ = false; |
250 return true; | 254 return true; |
251 } | 255 } |
252 | 256 |
253 REGISTER_MODULE_INITIALIZER(memfs_malloc, { | 257 REGISTER_MODULE_INITIALIZER(memfs_malloc, { |
254 if (FLAGS_memfs_malloc_path.length()) { | 258 if (FLAGS_memfs_malloc_path.length()) { |
255 SysAllocator* alloc = MallocExtension::instance()->GetSystemAllocator(); | 259 SysAllocator* alloc = MallocExtension::instance()->GetSystemAllocator(); |
256 HugetlbSysAllocator* hp = new (hugetlb_space) HugetlbSysAllocator(alloc); | 260 HugetlbSysAllocator* hp = new (hugetlb_space) HugetlbSysAllocator(alloc); |
257 if (hp->Initialize()) { | 261 if (hp->Initialize()) { |
258 MallocExtension::instance()->SetSystemAllocator(hp); | 262 MallocExtension::instance()->SetSystemAllocator(hp); |
259 } | 263 } |
260 } | 264 } |
261 }); | 265 }); |
262 | 266 |
263 #endif /* ifdef __linux */ | 267 #endif /* ifdef __linux */ |
OLD | NEW |