OLD | NEW |
1 // Copyright (c) 2005, Google Inc. | 1 // Copyright (c) 2005, Google Inc. |
2 // All rights reserved. | 2 // All rights reserved. |
3 // | 3 // |
4 // Redistribution and use in source and binary forms, with or without | 4 // Redistribution and use in source and binary forms, with or without |
5 // modification, are permitted provided that the following conditions are | 5 // modification, are permitted provided that the following conditions are |
6 // met: | 6 // met: |
7 // | 7 // |
8 // * Redistributions of source code must retain the above copyright | 8 // * Redistributions of source code must retain the above copyright |
9 // notice, this list of conditions and the following disclaimer. | 9 // notice, this list of conditions and the following disclaimer. |
10 // * Redistributions in binary form must reproduce the above | 10 // * Redistributions in binary form must reproduce the above |
(...skipping 28 matching lines...) Expand all Loading... |
39 # define mremap glibc_mremap | 39 # define mremap glibc_mremap |
40 # include <sys/mman.h> | 40 # include <sys/mman.h> |
41 # undef mremap | 41 # undef mremap |
42 #endif | 42 #endif |
43 | 43 |
44 #include <stddef.h> | 44 #include <stddef.h> |
45 #ifdef HAVE_STDINT_H | 45 #ifdef HAVE_STDINT_H |
46 #include <stdint.h> | 46 #include <stdint.h> |
47 #endif | 47 #endif |
48 #include <algorithm> | 48 #include <algorithm> |
| 49 #include "base/basictypes.h" |
49 #include "base/logging.h" | 50 #include "base/logging.h" |
50 #include "base/spinlock.h" | 51 #include "base/spinlock.h" |
51 #include "maybe_threads.h" | 52 #include "maybe_threads.h" |
52 #include "malloc_hook-inl.h" | 53 #include "malloc_hook-inl.h" |
53 #include <gperftools/malloc_hook.h> | 54 #include <google/malloc_hook.h> |
54 | 55 |
55 // This #ifdef should almost never be set. Set NO_TCMALLOC_SAMPLES if | 56 // This #ifdef should almost never be set. Set NO_TCMALLOC_SAMPLES if |
56 // you're porting to a system where you really can't get a stacktrace. | 57 // you're porting to a system where you really can't get a stacktrace. |
57 #ifdef NO_TCMALLOC_SAMPLES | 58 #ifdef NO_TCMALLOC_SAMPLES |
58 // We use #define so code compiles even if you #include stacktrace.h somehow. | 59 // We use #define so code compiles even if you #include stacktrace.h somehow. |
59 # define GetStackTrace(stack, depth, skip) (0) | 60 # define GetStackTrace(stack, depth, skip) (0) |
60 #else | 61 #else |
61 # include <gperftools/stacktrace.h> | 62 # include <google/stacktrace.h> |
62 #endif | 63 #endif |
63 | 64 |
64 // __THROW is defined in glibc systems. It means, counter-intuitively, | 65 // __THROW is defined in glibc systems. It means, counter-intuitively, |
65 // "This function will never throw an exception." It's an optional | 66 // "This function will never throw an exception." It's an optional |
66 // optimization tool, but we may need to use it to match glibc prototypes. | 67 // optimization tool, but we may need to use it to match glibc prototypes. |
67 #ifndef __THROW // I guess we're not on a glibc system | 68 #ifndef __THROW // I guess we're not on a glibc system |
68 # define __THROW // __THROW is just an optimization, so ok to make it "" | 69 # define __THROW // __THROW is just an optimization, so ok to make it "" |
69 #endif | 70 #endif |
70 | 71 |
71 using std::copy; | 72 using std::copy; |
(...skipping 124 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
196 // End of DEPRECATED code section. | 197 // End of DEPRECATED code section. |
197 | 198 |
198 // This lock is shared between all implementations of HookList::Add & Remove. | 199 // This lock is shared between all implementations of HookList::Add & Remove. |
199 // The potential for contention is very small. This needs to be a SpinLock and | 200 // The potential for contention is very small. This needs to be a SpinLock and |
200 // not a Mutex since it's possible for Mutex locking to allocate memory (e.g., | 201 // not a Mutex since it's possible for Mutex locking to allocate memory (e.g., |
201 // per-thread allocation in debug builds), which could cause infinite recursion. | 202 // per-thread allocation in debug builds), which could cause infinite recursion. |
202 static SpinLock hooklist_spinlock(base::LINKER_INITIALIZED); | 203 static SpinLock hooklist_spinlock(base::LINKER_INITIALIZED); |
203 | 204 |
204 template <typename T> | 205 template <typename T> |
205 bool HookList<T>::Add(T value_as_t) { | 206 bool HookList<T>::Add(T value_as_t) { |
206 AtomicWord value = bit_cast<AtomicWord>(value_as_t); | 207 // Note: we need to check this _before_ reinterpret_cast, since |
| 208 // reinterpret_cast may include random junk from memory. |
| 209 if (value_as_t == 0) { |
| 210 return false; |
| 211 } |
| 212 AtomicWord value = reinterpret_cast<const AtomicWord&>(value_as_t); |
207 if (value == 0) { | 213 if (value == 0) { |
| 214 // This should not actually happen, but just to be sure... |
208 return false; | 215 return false; |
209 } | 216 } |
210 SpinLockHolder l(&hooklist_spinlock); | 217 SpinLockHolder l(&hooklist_spinlock); |
211 // Find the first slot in data that is 0. | 218 // Find the first slot in data that is 0. |
212 int index = 0; | 219 int index = 0; |
213 while ((index < kHookListMaxValues) && | 220 while ((index < kHookListMaxValues) && |
214 (base::subtle::NoBarrier_Load(&priv_data[index]) != 0)) { | 221 (base::subtle::NoBarrier_Load(&priv_data[index]) != 0)) { |
215 ++index; | 222 ++index; |
216 } | 223 } |
217 if (index == kHookListMaxValues) { | 224 if (index == kHookListMaxValues) { |
218 return false; | 225 return false; |
219 } | 226 } |
220 AtomicWord prev_num_hooks = base::subtle::Acquire_Load(&priv_end); | 227 AtomicWord prev_num_hooks = base::subtle::Acquire_Load(&priv_end); |
221 base::subtle::Release_Store(&priv_data[index], value); | 228 base::subtle::Release_Store(&priv_data[index], value); |
222 if (prev_num_hooks <= index) { | 229 if (prev_num_hooks <= index) { |
223 base::subtle::Release_Store(&priv_end, index + 1); | 230 base::subtle::Release_Store(&priv_end, index + 1); |
224 } | 231 } |
225 return true; | 232 return true; |
226 } | 233 } |
227 | 234 |
228 template <typename T> | 235 template <typename T> |
229 bool HookList<T>::Remove(T value_as_t) { | 236 bool HookList<T>::Remove(T value_as_t) { |
230 if (value_as_t == 0) { | 237 if (value_as_t == 0) { |
231 return false; | 238 return false; |
232 } | 239 } |
233 SpinLockHolder l(&hooklist_spinlock); | 240 SpinLockHolder l(&hooklist_spinlock); |
234 AtomicWord hooks_end = base::subtle::Acquire_Load(&priv_end); | 241 AtomicWord hooks_end = base::subtle::Acquire_Load(&priv_end); |
235 int index = 0; | 242 int index = 0; |
236 while (index < hooks_end && value_as_t != bit_cast<T>( | 243 // Note: we need to cast back to T since T may be smaller than AtomicWord. |
| 244 while (index < hooks_end && value_as_t != reinterpret_cast<T>( |
237 base::subtle::Acquire_Load(&priv_data[index]))) { | 245 base::subtle::Acquire_Load(&priv_data[index]))) { |
238 ++index; | 246 ++index; |
239 } | 247 } |
240 if (index == hooks_end) { | 248 if (index == hooks_end) { |
241 return false; | 249 return false; |
242 } | 250 } |
243 base::subtle::Release_Store(&priv_data[index], 0); | 251 base::subtle::Release_Store(&priv_data[index], 0); |
244 if (hooks_end == index + 1) { | 252 if (hooks_end == index + 1) { |
245 // Adjust hooks_end down to the lowest possible value. | 253 // Adjust hooks_end down to the lowest possible value. |
246 hooks_end = index; | 254 hooks_end = index; |
247 while ((hooks_end > 0) && | 255 while ((hooks_end > 0) && |
248 (base::subtle::Acquire_Load(&priv_data[hooks_end - 1]) == 0)) { | 256 (base::subtle::Acquire_Load(&priv_data[hooks_end - 1]) == 0)) { |
249 --hooks_end; | 257 --hooks_end; |
250 } | 258 } |
251 base::subtle::Release_Store(&priv_end, hooks_end); | 259 base::subtle::Release_Store(&priv_end, hooks_end); |
252 } | 260 } |
253 return true; | 261 return true; |
254 } | 262 } |
255 | 263 |
256 template <typename T> | 264 template <typename T> |
257 int HookList<T>::Traverse(T* output_array, int n) const { | 265 int HookList<T>::Traverse(T* output_array, int n) const { |
258 AtomicWord hooks_end = base::subtle::Acquire_Load(&priv_end); | 266 AtomicWord hooks_end = base::subtle::Acquire_Load(&priv_end); |
259 int actual_hooks_end = 0; | 267 int actual_hooks_end = 0; |
260 for (int i = 0; i < hooks_end && n > 0; ++i) { | 268 for (int i = 0; i < hooks_end && n > 0; ++i) { |
261 AtomicWord data = base::subtle::Acquire_Load(&priv_data[i]); | 269 AtomicWord data = base::subtle::Acquire_Load(&priv_data[i]); |
262 if (data != 0) { | 270 if (data != 0) { |
263 *output_array++ = bit_cast<T>(data); | 271 *output_array++ = reinterpret_cast<const T&>(data); |
264 ++actual_hooks_end; | 272 ++actual_hooks_end; |
265 --n; | 273 --n; |
266 } | 274 } |
267 } | 275 } |
268 return actual_hooks_end; | 276 return actual_hooks_end; |
269 } | 277 } |
270 | 278 |
271 // Initialize a HookList (optionally with the given initial_value in index 0). | 279 // Initialize a HookList (optionally with the given initial_value in index 0). |
272 #define INIT_HOOK_LIST { 0 } | 280 #define INIT_HOOK_LIST { 0 } |
273 #define INIT_HOOK_LIST_WITH_VALUE(initial_value) \ | 281 #define INIT_HOOK_LIST_WITH_VALUE(initial_value) \ |
274 { 1, { reinterpret_cast<AtomicWord>(initial_value) } } | 282 { 1, { reinterpret_cast<AtomicWord>(initial_value) } } |
275 | 283 |
276 // Explicit instantiation for malloc_hook_test.cc. This ensures all the methods | 284 // Explicit instantiation for malloc_hook_test.cc. This ensures all the methods |
277 // are instantiated. | 285 // are instantiated. |
278 template struct HookList<MallocHook::NewHook>; | 286 template class HookList<MallocHook::NewHook>; |
279 | 287 |
280 HookList<MallocHook::NewHook> new_hooks_ = | 288 HookList<MallocHook::NewHook> new_hooks_ = |
281 INIT_HOOK_LIST_WITH_VALUE(&InitialNewHook); | 289 INIT_HOOK_LIST_WITH_VALUE(&InitialNewHook); |
282 HookList<MallocHook::DeleteHook> delete_hooks_ = INIT_HOOK_LIST; | 290 HookList<MallocHook::DeleteHook> delete_hooks_ = INIT_HOOK_LIST; |
283 HookList<MallocHook::PreMmapHook> premmap_hooks_ = | 291 HookList<MallocHook::PreMmapHook> premmap_hooks_ = |
284 INIT_HOOK_LIST_WITH_VALUE(&InitialPreMMapHook); | 292 INIT_HOOK_LIST_WITH_VALUE(&InitialPreMMapHook); |
285 HookList<MallocHook::MmapHook> mmap_hooks_ = INIT_HOOK_LIST; | 293 HookList<MallocHook::MmapHook> mmap_hooks_ = INIT_HOOK_LIST; |
286 HookList<MallocHook::MunmapHook> munmap_hooks_ = INIT_HOOK_LIST; | 294 HookList<MallocHook::MunmapHook> munmap_hooks_ = INIT_HOOK_LIST; |
287 HookList<MallocHook::MremapHook> mremap_hooks_ = INIT_HOOK_LIST; | 295 HookList<MallocHook::MremapHook> mremap_hooks_ = INIT_HOOK_LIST; |
288 HookList<MallocHook::PreSbrkHook> presbrk_hooks_ = | 296 HookList<MallocHook::PreSbrkHook> presbrk_hooks_ = |
(...skipping 394 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
683 } | 691 } |
684 RAW_LOG(WARNING, "Hooked allocator frame not found, returning empty trace"); | 692 RAW_LOG(WARNING, "Hooked allocator frame not found, returning empty trace"); |
685 // If this happens try increasing kMaxSkip | 693 // If this happens try increasing kMaxSkip |
686 // or else something must be wrong with InHookCaller, | 694 // or else something must be wrong with InHookCaller, |
687 // e.g. for every section used in InHookCaller | 695 // e.g. for every section used in InHookCaller |
688 // all functions in that section must be inside the same library. | 696 // all functions in that section must be inside the same library. |
689 return 0; | 697 return 0; |
690 #endif | 698 #endif |
691 } | 699 } |
692 | 700 |
693 // On systems where we know how, we override mmap/munmap/mremap/sbrk | 701 // On Linux/x86, we override mmap/munmap/mremap/sbrk |
694 // to provide support for calling the related hooks (in addition, | 702 // and provide support for calling the related hooks. |
695 // of course, to doing what these functions normally do). | 703 // |
| 704 // We define mmap() and mmap64(), which somewhat reimplements libc's mmap |
| 705 // syscall stubs. Unfortunately libc only exports the stubs via weak symbols |
| 706 // (which we're overriding with our mmap64() and mmap() wrappers) so we can't |
| 707 // just call through to them. |
696 | 708 |
697 #if defined(__linux) | |
698 # include "malloc_hook_mmap_linux.h" | |
699 | 709 |
700 #elif defined(__FreeBSD__) | 710 #if defined(__linux) && \ |
701 # include "malloc_hook_mmap_freebsd.h" | 711 (defined(__i386__) || defined(__x86_64__) || defined(__PPC__)) |
| 712 #include <unistd.h> |
| 713 #include <syscall.h> |
| 714 #include <sys/mman.h> |
| 715 #include <errno.h> |
| 716 #include "base/linux_syscall_support.h" |
702 | 717 |
703 #else | 718 // The x86-32 case and the x86-64 case differ: |
| 719 // 32b has a mmap2() syscall, 64b does not. |
| 720 // 64b and 32b have different calling conventions for mmap(). |
| 721 #if defined(__x86_64__) || defined(__PPC64__) |
| 722 |
| 723 static inline void* do_mmap64(void *start, size_t length, |
| 724 int prot, int flags, |
| 725 int fd, __off64_t offset) __THROW { |
| 726 return (void *)syscall(SYS_mmap, start, length, prot, flags, fd, offset); |
| 727 } |
| 728 |
| 729 #elif defined(__i386__) || defined(__PPC__) |
| 730 |
| 731 static inline void* do_mmap64(void *start, size_t length, |
| 732 int prot, int flags, |
| 733 int fd, __off64_t offset) __THROW { |
| 734 void *result; |
| 735 |
| 736 // Try mmap2() unless it's not supported |
| 737 static bool have_mmap2 = true; |
| 738 if (have_mmap2) { |
| 739 static int pagesize = 0; |
| 740 if (!pagesize) pagesize = getpagesize(); |
| 741 |
| 742 // Check that the offset is page aligned |
| 743 if (offset & (pagesize - 1)) { |
| 744 result = MAP_FAILED; |
| 745 errno = EINVAL; |
| 746 goto out; |
| 747 } |
| 748 |
| 749 result = (void *)syscall(SYS_mmap2, |
| 750 start, length, prot, flags, fd, |
| 751 (off_t) (offset / pagesize)); |
| 752 if (result != MAP_FAILED || errno != ENOSYS) goto out; |
| 753 |
| 754 // We don't have mmap2() after all - don't bother trying it in future |
| 755 have_mmap2 = false; |
| 756 } |
| 757 |
| 758 if (((off_t)offset) != offset) { |
| 759 // If we're trying to map a 64-bit offset, fail now since we don't |
| 760 // have 64-bit mmap() support. |
| 761 result = MAP_FAILED; |
| 762 errno = EINVAL; |
| 763 goto out; |
| 764 } |
| 765 |
| 766 { |
| 767 // Fall back to old 32-bit offset mmap() call |
| 768 // Old syscall interface cannot handle six args, so pass in an array |
| 769 int32 args[6] = { (int32) start, length, prot, flags, fd, (off_t) offset }; |
| 770 result = (void *)syscall(SYS_mmap, args); |
| 771 } |
| 772 out: |
| 773 return result; |
| 774 } |
| 775 |
| 776 # endif // defined(__x86_64__) |
| 777 |
| 778 // We use do_mmap64 abstraction to put MallocHook::InvokeMmapHook |
| 779 // calls right into mmap and mmap64, so that the stack frames in the caller's |
| 780 // stack are at the same offsets for all the calls of memory allocating |
| 781 // functions. |
| 782 |
| 783 // Put all callers of MallocHook::Invoke* in this module into |
| 784 // malloc_hook section, |
| 785 // so that MallocHook::GetCallerStackTrace can function accurately: |
| 786 |
| 787 // Make sure mmap doesn't get #define'd away by <sys/mman.h> |
| 788 #undef mmap |
| 789 |
| 790 extern "C" { |
| 791 void* mmap64(void *start, size_t length, int prot, int flags, |
| 792 int fd, __off64_t offset ) __THROW |
| 793 ATTRIBUTE_SECTION(malloc_hook); |
| 794 void* mmap(void *start, size_t length,int prot, int flags, |
| 795 int fd, off_t offset) __THROW |
| 796 ATTRIBUTE_SECTION(malloc_hook); |
| 797 int munmap(void* start, size_t length) __THROW |
| 798 ATTRIBUTE_SECTION(malloc_hook); |
| 799 void* mremap(void* old_addr, size_t old_size, size_t new_size, |
| 800 int flags, ...) __THROW |
| 801 ATTRIBUTE_SECTION(malloc_hook); |
| 802 void* sbrk(std::ptrdiff_t increment) __THROW |
| 803 ATTRIBUTE_SECTION(malloc_hook); |
| 804 } |
| 805 |
| 806 extern "C" void* mmap64(void *start, size_t length, int prot, int flags, |
| 807 int fd, __off64_t offset) __THROW { |
| 808 MallocHook::InvokePreMmapHook(start, length, prot, flags, fd, offset); |
| 809 void *result; |
| 810 if (!MallocHook::InvokeMmapReplacement( |
| 811 start, length, prot, flags, fd, offset, &result)) { |
| 812 result = do_mmap64(start, length, prot, flags, fd, offset); |
| 813 } |
| 814 MallocHook::InvokeMmapHook(result, start, length, prot, flags, fd, offset); |
| 815 return result; |
| 816 } |
| 817 |
| 818 #if !defined(__USE_FILE_OFFSET64) || !defined(__REDIRECT_NTH) |
| 819 |
| 820 extern "C" void* mmap(void *start, size_t length, int prot, int flags, |
| 821 int fd, off_t offset) __THROW { |
| 822 MallocHook::InvokePreMmapHook(start, length, prot, flags, fd, offset); |
| 823 void *result; |
| 824 if (!MallocHook::InvokeMmapReplacement( |
| 825 start, length, prot, flags, fd, offset, &result)) { |
| 826 result = do_mmap64(start, length, prot, flags, fd, |
| 827 static_cast<size_t>(offset)); // avoid sign extension |
| 828 } |
| 829 MallocHook::InvokeMmapHook(result, start, length, prot, flags, fd, offset); |
| 830 return result; |
| 831 } |
| 832 |
| 833 #endif // !defined(__USE_FILE_OFFSET64) || !defined(__REDIRECT_NTH) |
| 834 |
| 835 extern "C" int munmap(void* start, size_t length) __THROW { |
| 836 MallocHook::InvokeMunmapHook(start, length); |
| 837 int result; |
| 838 if (!MallocHook::InvokeMunmapReplacement(start, length, &result)) { |
| 839 result = syscall(SYS_munmap, start, length); |
| 840 } |
| 841 return result; |
| 842 } |
| 843 |
| 844 extern "C" void* mremap(void* old_addr, size_t old_size, size_t new_size, |
| 845 int flags, ...) __THROW { |
| 846 va_list ap; |
| 847 va_start(ap, flags); |
| 848 void *new_address = va_arg(ap, void *); |
| 849 va_end(ap); |
| 850 void* result = sys_mremap(old_addr, old_size, new_size, flags, new_address); |
| 851 MallocHook::InvokeMremapHook(result, old_addr, old_size, new_size, flags, |
| 852 new_address); |
| 853 return result; |
| 854 } |
| 855 |
| 856 // libc's version: |
| 857 extern "C" void* __sbrk(std::ptrdiff_t increment); |
| 858 |
| 859 extern "C" void* sbrk(std::ptrdiff_t increment) __THROW { |
| 860 MallocHook::InvokePreSbrkHook(increment); |
| 861 void *result = __sbrk(increment); |
| 862 MallocHook::InvokeSbrkHook(result, increment); |
| 863 return result; |
| 864 } |
| 865 |
| 866 /*static*/void* MallocHook::UnhookedMMap(void *start, size_t length, int prot, |
| 867 int flags, int fd, off_t offset) { |
| 868 void* result; |
| 869 if (!MallocHook::InvokeMmapReplacement( |
| 870 start, length, prot, flags, fd, offset, &result)) { |
| 871 result = do_mmap64(start, length, prot, flags, fd, offset); |
| 872 } |
| 873 return result; |
| 874 } |
| 875 |
| 876 /*static*/int MallocHook::UnhookedMUnmap(void *start, size_t length) { |
| 877 int result; |
| 878 if (!MallocHook::InvokeMunmapReplacement(start, length, &result)) { |
| 879 result = sys_munmap(start, length); |
| 880 } |
| 881 return result; |
| 882 } |
| 883 |
| 884 #else // defined(__linux) && |
| 885 // (defined(__i386__) || defined(__x86_64__) || defined(__PPC__)) |
704 | 886 |
705 /*static*/void* MallocHook::UnhookedMMap(void *start, size_t length, int prot, | 887 /*static*/void* MallocHook::UnhookedMMap(void *start, size_t length, int prot, |
706 int flags, int fd, off_t offset) { | 888 int flags, int fd, off_t offset) { |
707 void* result; | 889 void* result; |
708 if (!MallocHook::InvokeMmapReplacement( | 890 if (!MallocHook::InvokeMmapReplacement( |
709 start, length, prot, flags, fd, offset, &result)) { | 891 start, length, prot, flags, fd, offset, &result)) { |
710 result = mmap(start, length, prot, flags, fd, offset); | 892 result = mmap(start, length, prot, flags, fd, offset); |
711 } | 893 } |
712 return result; | 894 return result; |
713 } | 895 } |
714 | 896 |
715 /*static*/int MallocHook::UnhookedMUnmap(void *start, size_t length) { | 897 /*static*/int MallocHook::UnhookedMUnmap(void *start, size_t length) { |
716 int result; | 898 int result; |
717 if (!MallocHook::InvokeMunmapReplacement(start, length, &result)) { | 899 if (!MallocHook::InvokeMunmapReplacement(start, length, &result)) { |
718 result = munmap(start, length); | 900 result = munmap(start, length); |
719 } | 901 } |
720 return result; | 902 return result; |
721 } | 903 } |
722 | 904 |
723 #endif | 905 #endif // defined(__linux) && |
| 906 // (defined(__i386__) || defined(__x86_64__) || defined(__PPC__)) |
OLD | NEW |