OLD | NEW |
1 // Copyright (c) 2005, Google Inc. | 1 // Copyright (c) 2005, Google Inc. |
2 // All rights reserved. | 2 // All rights reserved. |
3 // | 3 // |
4 // Redistribution and use in source and binary forms, with or without | 4 // Redistribution and use in source and binary forms, with or without |
5 // modification, are permitted provided that the following conditions are | 5 // modification, are permitted provided that the following conditions are |
6 // met: | 6 // met: |
7 // | 7 // |
8 // * Redistributions of source code must retain the above copyright | 8 // * Redistributions of source code must retain the above copyright |
9 // notice, this list of conditions and the following disclaimer. | 9 // notice, this list of conditions and the following disclaimer. |
10 // * Redistributions in binary form must reproduce the above | 10 // * Redistributions in binary form must reproduce the above |
(...skipping 28 matching lines...) Expand all Loading... |
39 # define mremap glibc_mremap | 39 # define mremap glibc_mremap |
40 # include <sys/mman.h> | 40 # include <sys/mman.h> |
41 # undef mremap | 41 # undef mremap |
42 #endif | 42 #endif |
43 | 43 |
44 #include <stddef.h> | 44 #include <stddef.h> |
45 #ifdef HAVE_STDINT_H | 45 #ifdef HAVE_STDINT_H |
46 #include <stdint.h> | 46 #include <stdint.h> |
47 #endif | 47 #endif |
48 #include <algorithm> | 48 #include <algorithm> |
49 #include "base/basictypes.h" | |
50 #include "base/logging.h" | 49 #include "base/logging.h" |
51 #include "base/spinlock.h" | 50 #include "base/spinlock.h" |
52 #include "maybe_threads.h" | 51 #include "maybe_threads.h" |
53 #include "malloc_hook-inl.h" | 52 #include "malloc_hook-inl.h" |
54 #include <google/malloc_hook.h> | 53 #include <gperftools/malloc_hook.h> |
55 | 54 |
56 // This #ifdef should almost never be set. Set NO_TCMALLOC_SAMPLES if | 55 // This #ifdef should almost never be set. Set NO_TCMALLOC_SAMPLES if |
57 // you're porting to a system where you really can't get a stacktrace. | 56 // you're porting to a system where you really can't get a stacktrace. |
58 #ifdef NO_TCMALLOC_SAMPLES | 57 #ifdef NO_TCMALLOC_SAMPLES |
59 // We use #define so code compiles even if you #include stacktrace.h somehow. | 58 // We use #define so code compiles even if you #include stacktrace.h somehow. |
60 # define GetStackTrace(stack, depth, skip) (0) | 59 # define GetStackTrace(stack, depth, skip) (0) |
61 #else | 60 #else |
62 # include <google/stacktrace.h> | 61 # include <gperftools/stacktrace.h> |
63 #endif | 62 #endif |
64 | 63 |
65 // __THROW is defined in glibc systems. It means, counter-intuitively, | 64 // __THROW is defined in glibc systems. It means, counter-intuitively, |
66 // "This function will never throw an exception." It's an optional | 65 // "This function will never throw an exception." It's an optional |
67 // optimization tool, but we may need to use it to match glibc prototypes. | 66 // optimization tool, but we may need to use it to match glibc prototypes. |
68 #ifndef __THROW // I guess we're not on a glibc system | 67 #ifndef __THROW // I guess we're not on a glibc system |
69 # define __THROW // __THROW is just an optimization, so ok to make it "" | 68 # define __THROW // __THROW is just an optimization, so ok to make it "" |
70 #endif | 69 #endif |
71 | 70 |
72 using std::copy; | 71 using std::copy; |
(...skipping 124 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
197 // End of DEPRECATED code section. | 196 // End of DEPRECATED code section. |
198 | 197 |
199 // This lock is shared between all implementations of HookList::Add & Remove. | 198 // This lock is shared between all implementations of HookList::Add & Remove. |
200 // The potential for contention is very small. This needs to be a SpinLock and | 199 // The potential for contention is very small. This needs to be a SpinLock and |
201 // not a Mutex since it's possible for Mutex locking to allocate memory (e.g., | 200 // not a Mutex since it's possible for Mutex locking to allocate memory (e.g., |
202 // per-thread allocation in debug builds), which could cause infinite recursion. | 201 // per-thread allocation in debug builds), which could cause infinite recursion. |
203 static SpinLock hooklist_spinlock(base::LINKER_INITIALIZED); | 202 static SpinLock hooklist_spinlock(base::LINKER_INITIALIZED); |
204 | 203 |
205 template <typename T> | 204 template <typename T> |
206 bool HookList<T>::Add(T value_as_t) { | 205 bool HookList<T>::Add(T value_as_t) { |
207 // Note: we need to check this _before_ reinterpret_cast, since | 206 AtomicWord value = bit_cast<AtomicWord>(value_as_t); |
208 // reinterpret_cast may include random junk from memory. | |
209 if (value_as_t == 0) { | |
210 return false; | |
211 } | |
212 AtomicWord value = reinterpret_cast<const AtomicWord&>(value_as_t); | |
213 if (value == 0) { | 207 if (value == 0) { |
214 // This should not actually happen, but just to be sure... | |
215 return false; | 208 return false; |
216 } | 209 } |
217 SpinLockHolder l(&hooklist_spinlock); | 210 SpinLockHolder l(&hooklist_spinlock); |
218 // Find the first slot in data that is 0. | 211 // Find the first slot in data that is 0. |
219 int index = 0; | 212 int index = 0; |
220 while ((index < kHookListMaxValues) && | 213 while ((index < kHookListMaxValues) && |
221 (base::subtle::NoBarrier_Load(&priv_data[index]) != 0)) { | 214 (base::subtle::NoBarrier_Load(&priv_data[index]) != 0)) { |
222 ++index; | 215 ++index; |
223 } | 216 } |
224 if (index == kHookListMaxValues) { | 217 if (index == kHookListMaxValues) { |
225 return false; | 218 return false; |
226 } | 219 } |
227 AtomicWord prev_num_hooks = base::subtle::Acquire_Load(&priv_end); | 220 AtomicWord prev_num_hooks = base::subtle::Acquire_Load(&priv_end); |
228 base::subtle::Release_Store(&priv_data[index], value); | 221 base::subtle::Release_Store(&priv_data[index], value); |
229 if (prev_num_hooks <= index) { | 222 if (prev_num_hooks <= index) { |
230 base::subtle::Release_Store(&priv_end, index + 1); | 223 base::subtle::Release_Store(&priv_end, index + 1); |
231 } | 224 } |
232 return true; | 225 return true; |
233 } | 226 } |
234 | 227 |
235 template <typename T> | 228 template <typename T> |
236 bool HookList<T>::Remove(T value_as_t) { | 229 bool HookList<T>::Remove(T value_as_t) { |
237 if (value_as_t == 0) { | 230 if (value_as_t == 0) { |
238 return false; | 231 return false; |
239 } | 232 } |
240 SpinLockHolder l(&hooklist_spinlock); | 233 SpinLockHolder l(&hooklist_spinlock); |
241 AtomicWord hooks_end = base::subtle::Acquire_Load(&priv_end); | 234 AtomicWord hooks_end = base::subtle::Acquire_Load(&priv_end); |
242 int index = 0; | 235 int index = 0; |
243 // Note: we need to cast back to T since T may be smaller than AtomicWord. | 236 while (index < hooks_end && value_as_t != bit_cast<T>( |
244 while (index < hooks_end && value_as_t != reinterpret_cast<T>( | |
245 base::subtle::Acquire_Load(&priv_data[index]))) { | 237 base::subtle::Acquire_Load(&priv_data[index]))) { |
246 ++index; | 238 ++index; |
247 } | 239 } |
248 if (index == hooks_end) { | 240 if (index == hooks_end) { |
249 return false; | 241 return false; |
250 } | 242 } |
251 base::subtle::Release_Store(&priv_data[index], 0); | 243 base::subtle::Release_Store(&priv_data[index], 0); |
252 if (hooks_end == index + 1) { | 244 if (hooks_end == index + 1) { |
253 // Adjust hooks_end down to the lowest possible value. | 245 // Adjust hooks_end down to the lowest possible value. |
254 hooks_end = index; | 246 hooks_end = index; |
255 while ((hooks_end > 0) && | 247 while ((hooks_end > 0) && |
256 (base::subtle::Acquire_Load(&priv_data[hooks_end - 1]) == 0)) { | 248 (base::subtle::Acquire_Load(&priv_data[hooks_end - 1]) == 0)) { |
257 --hooks_end; | 249 --hooks_end; |
258 } | 250 } |
259 base::subtle::Release_Store(&priv_end, hooks_end); | 251 base::subtle::Release_Store(&priv_end, hooks_end); |
260 } | 252 } |
261 return true; | 253 return true; |
262 } | 254 } |
263 | 255 |
264 template <typename T> | 256 template <typename T> |
265 int HookList<T>::Traverse(T* output_array, int n) const { | 257 int HookList<T>::Traverse(T* output_array, int n) const { |
266 AtomicWord hooks_end = base::subtle::Acquire_Load(&priv_end); | 258 AtomicWord hooks_end = base::subtle::Acquire_Load(&priv_end); |
267 int actual_hooks_end = 0; | 259 int actual_hooks_end = 0; |
268 for (int i = 0; i < hooks_end && n > 0; ++i) { | 260 for (int i = 0; i < hooks_end && n > 0; ++i) { |
269 AtomicWord data = base::subtle::Acquire_Load(&priv_data[i]); | 261 AtomicWord data = base::subtle::Acquire_Load(&priv_data[i]); |
270 if (data != 0) { | 262 if (data != 0) { |
271 *output_array++ = reinterpret_cast<const T&>(data); | 263 *output_array++ = bit_cast<T>(data); |
272 ++actual_hooks_end; | 264 ++actual_hooks_end; |
273 --n; | 265 --n; |
274 } | 266 } |
275 } | 267 } |
276 return actual_hooks_end; | 268 return actual_hooks_end; |
277 } | 269 } |
278 | 270 |
279 // Initialize a HookList (optionally with the given initial_value in index 0). | 271 // Initialize a HookList (optionally with the given initial_value in index 0). |
280 #define INIT_HOOK_LIST { 0 } | 272 #define INIT_HOOK_LIST { 0 } |
281 #define INIT_HOOK_LIST_WITH_VALUE(initial_value) \ | 273 #define INIT_HOOK_LIST_WITH_VALUE(initial_value) \ |
282 { 1, { reinterpret_cast<AtomicWord>(initial_value) } } | 274 { 1, { reinterpret_cast<AtomicWord>(initial_value) } } |
283 | 275 |
284 // Explicit instantiation for malloc_hook_test.cc. This ensures all the methods | 276 // Explicit instantiation for malloc_hook_test.cc. This ensures all the methods |
285 // are instantiated. | 277 // are instantiated. |
286 template class HookList<MallocHook::NewHook>; | 278 template struct HookList<MallocHook::NewHook>; |
287 | 279 |
288 HookList<MallocHook::NewHook> new_hooks_ = | 280 HookList<MallocHook::NewHook> new_hooks_ = |
289 INIT_HOOK_LIST_WITH_VALUE(&InitialNewHook); | 281 INIT_HOOK_LIST_WITH_VALUE(&InitialNewHook); |
290 HookList<MallocHook::DeleteHook> delete_hooks_ = INIT_HOOK_LIST; | 282 HookList<MallocHook::DeleteHook> delete_hooks_ = INIT_HOOK_LIST; |
291 HookList<MallocHook::PreMmapHook> premmap_hooks_ = | 283 HookList<MallocHook::PreMmapHook> premmap_hooks_ = |
292 INIT_HOOK_LIST_WITH_VALUE(&InitialPreMMapHook); | 284 INIT_HOOK_LIST_WITH_VALUE(&InitialPreMMapHook); |
293 HookList<MallocHook::MmapHook> mmap_hooks_ = INIT_HOOK_LIST; | 285 HookList<MallocHook::MmapHook> mmap_hooks_ = INIT_HOOK_LIST; |
294 HookList<MallocHook::MunmapHook> munmap_hooks_ = INIT_HOOK_LIST; | 286 HookList<MallocHook::MunmapHook> munmap_hooks_ = INIT_HOOK_LIST; |
295 HookList<MallocHook::MremapHook> mremap_hooks_ = INIT_HOOK_LIST; | 287 HookList<MallocHook::MremapHook> mremap_hooks_ = INIT_HOOK_LIST; |
296 HookList<MallocHook::PreSbrkHook> presbrk_hooks_ = | 288 HookList<MallocHook::PreSbrkHook> presbrk_hooks_ = |
(...skipping 394 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
691 } | 683 } |
692 RAW_LOG(WARNING, "Hooked allocator frame not found, returning empty trace"); | 684 RAW_LOG(WARNING, "Hooked allocator frame not found, returning empty trace"); |
693 // If this happens try increasing kMaxSkip | 685 // If this happens try increasing kMaxSkip |
694 // or else something must be wrong with InHookCaller, | 686 // or else something must be wrong with InHookCaller, |
695 // e.g. for every section used in InHookCaller | 687 // e.g. for every section used in InHookCaller |
696 // all functions in that section must be inside the same library. | 688 // all functions in that section must be inside the same library. |
697 return 0; | 689 return 0; |
698 #endif | 690 #endif |
699 } | 691 } |
700 | 692 |
701 // On Linux/x86, we override mmap/munmap/mremap/sbrk | 693 // On systems where we know how, we override mmap/munmap/mremap/sbrk |
702 // and provide support for calling the related hooks. | 694 // to provide support for calling the related hooks (in addition, |
703 // | 695 // of course, to doing what these functions normally do). |
704 // We define mmap() and mmap64(), which somewhat reimplements libc's mmap | |
705 // syscall stubs. Unfortunately libc only exports the stubs via weak symbols | |
706 // (which we're overriding with our mmap64() and mmap() wrappers) so we can't | |
707 // just call through to them. | |
708 | 696 |
| 697 #if defined(__linux) |
| 698 # include "malloc_hook_mmap_linux.h" |
709 | 699 |
710 #if defined(__linux) && \ | 700 #elif defined(__FreeBSD__) |
711 (defined(__i386__) || defined(__x86_64__) || defined(__PPC__)) | 701 # include "malloc_hook_mmap_freebsd.h" |
712 #include <unistd.h> | |
713 #include <syscall.h> | |
714 #include <sys/mman.h> | |
715 #include <errno.h> | |
716 #include "base/linux_syscall_support.h" | |
717 | 702 |
718 // The x86-32 case and the x86-64 case differ: | 703 #else |
719 // 32b has a mmap2() syscall, 64b does not. | |
720 // 64b and 32b have different calling conventions for mmap(). | |
721 #if defined(__x86_64__) || defined(__PPC64__) | |
722 | |
723 static inline void* do_mmap64(void *start, size_t length, | |
724 int prot, int flags, | |
725 int fd, __off64_t offset) __THROW { | |
726 return (void *)syscall(SYS_mmap, start, length, prot, flags, fd, offset); | |
727 } | |
728 | |
729 #elif defined(__i386__) || defined(__PPC__) | |
730 | |
731 static inline void* do_mmap64(void *start, size_t length, | |
732 int prot, int flags, | |
733 int fd, __off64_t offset) __THROW { | |
734 void *result; | |
735 | |
736 // Try mmap2() unless it's not supported | |
737 static bool have_mmap2 = true; | |
738 if (have_mmap2) { | |
739 static int pagesize = 0; | |
740 if (!pagesize) pagesize = getpagesize(); | |
741 | |
742 // Check that the offset is page aligned | |
743 if (offset & (pagesize - 1)) { | |
744 result = MAP_FAILED; | |
745 errno = EINVAL; | |
746 goto out; | |
747 } | |
748 | |
749 result = (void *)syscall(SYS_mmap2, | |
750 start, length, prot, flags, fd, | |
751 (off_t) (offset / pagesize)); | |
752 if (result != MAP_FAILED || errno != ENOSYS) goto out; | |
753 | |
754 // We don't have mmap2() after all - don't bother trying it in future | |
755 have_mmap2 = false; | |
756 } | |
757 | |
758 if (((off_t)offset) != offset) { | |
759 // If we're trying to map a 64-bit offset, fail now since we don't | |
760 // have 64-bit mmap() support. | |
761 result = MAP_FAILED; | |
762 errno = EINVAL; | |
763 goto out; | |
764 } | |
765 | |
766 { | |
767 // Fall back to old 32-bit offset mmap() call | |
768 // Old syscall interface cannot handle six args, so pass in an array | |
769 int32 args[6] = { (int32) start, length, prot, flags, fd, (off_t) offset }; | |
770 result = (void *)syscall(SYS_mmap, args); | |
771 } | |
772 out: | |
773 return result; | |
774 } | |
775 | |
776 # endif // defined(__x86_64__) | |
777 | |
778 // We use do_mmap64 abstraction to put MallocHook::InvokeMmapHook | |
779 // calls right into mmap and mmap64, so that the stack frames in the caller's | |
780 // stack are at the same offsets for all the calls of memory allocating | |
781 // functions. | |
782 | |
783 // Put all callers of MallocHook::Invoke* in this module into | |
784 // malloc_hook section, | |
785 // so that MallocHook::GetCallerStackTrace can function accurately: | |
786 | |
787 // Make sure mmap doesn't get #define'd away by <sys/mman.h> | |
788 #undef mmap | |
789 | |
790 extern "C" { | |
791 void* mmap64(void *start, size_t length, int prot, int flags, | |
792 int fd, __off64_t offset ) __THROW | |
793 ATTRIBUTE_SECTION(malloc_hook); | |
794 void* mmap(void *start, size_t length,int prot, int flags, | |
795 int fd, off_t offset) __THROW | |
796 ATTRIBUTE_SECTION(malloc_hook); | |
797 int munmap(void* start, size_t length) __THROW | |
798 ATTRIBUTE_SECTION(malloc_hook); | |
799 void* mremap(void* old_addr, size_t old_size, size_t new_size, | |
800 int flags, ...) __THROW | |
801 ATTRIBUTE_SECTION(malloc_hook); | |
802 void* sbrk(std::ptrdiff_t increment) __THROW | |
803 ATTRIBUTE_SECTION(malloc_hook); | |
804 } | |
805 | |
806 extern "C" void* mmap64(void *start, size_t length, int prot, int flags, | |
807 int fd, __off64_t offset) __THROW { | |
808 MallocHook::InvokePreMmapHook(start, length, prot, flags, fd, offset); | |
809 void *result; | |
810 if (!MallocHook::InvokeMmapReplacement( | |
811 start, length, prot, flags, fd, offset, &result)) { | |
812 result = do_mmap64(start, length, prot, flags, fd, offset); | |
813 } | |
814 MallocHook::InvokeMmapHook(result, start, length, prot, flags, fd, offset); | |
815 return result; | |
816 } | |
817 | |
818 #if !defined(__USE_FILE_OFFSET64) || !defined(__REDIRECT_NTH) | |
819 | |
820 extern "C" void* mmap(void *start, size_t length, int prot, int flags, | |
821 int fd, off_t offset) __THROW { | |
822 MallocHook::InvokePreMmapHook(start, length, prot, flags, fd, offset); | |
823 void *result; | |
824 if (!MallocHook::InvokeMmapReplacement( | |
825 start, length, prot, flags, fd, offset, &result)) { | |
826 result = do_mmap64(start, length, prot, flags, fd, | |
827 static_cast<size_t>(offset)); // avoid sign extension | |
828 } | |
829 MallocHook::InvokeMmapHook(result, start, length, prot, flags, fd, offset); | |
830 return result; | |
831 } | |
832 | |
833 #endif // !defined(__USE_FILE_OFFSET64) || !defined(__REDIRECT_NTH) | |
834 | |
835 extern "C" int munmap(void* start, size_t length) __THROW { | |
836 MallocHook::InvokeMunmapHook(start, length); | |
837 int result; | |
838 if (!MallocHook::InvokeMunmapReplacement(start, length, &result)) { | |
839 result = syscall(SYS_munmap, start, length); | |
840 } | |
841 return result; | |
842 } | |
843 | |
844 extern "C" void* mremap(void* old_addr, size_t old_size, size_t new_size, | |
845 int flags, ...) __THROW { | |
846 va_list ap; | |
847 va_start(ap, flags); | |
848 void *new_address = va_arg(ap, void *); | |
849 va_end(ap); | |
850 void* result = sys_mremap(old_addr, old_size, new_size, flags, new_address); | |
851 MallocHook::InvokeMremapHook(result, old_addr, old_size, new_size, flags, | |
852 new_address); | |
853 return result; | |
854 } | |
855 | |
856 // libc's version: | |
857 extern "C" void* __sbrk(std::ptrdiff_t increment); | |
858 | |
859 extern "C" void* sbrk(std::ptrdiff_t increment) __THROW { | |
860 MallocHook::InvokePreSbrkHook(increment); | |
861 void *result = __sbrk(increment); | |
862 MallocHook::InvokeSbrkHook(result, increment); | |
863 return result; | |
864 } | |
865 | 704 |
866 /*static*/void* MallocHook::UnhookedMMap(void *start, size_t length, int prot, | 705 /*static*/void* MallocHook::UnhookedMMap(void *start, size_t length, int prot, |
867 int flags, int fd, off_t offset) { | 706 int flags, int fd, off_t offset) { |
868 void* result; | |
869 if (!MallocHook::InvokeMmapReplacement( | |
870 start, length, prot, flags, fd, offset, &result)) { | |
871 result = do_mmap64(start, length, prot, flags, fd, offset); | |
872 } | |
873 return result; | |
874 } | |
875 | |
876 /*static*/int MallocHook::UnhookedMUnmap(void *start, size_t length) { | |
877 int result; | |
878 if (!MallocHook::InvokeMunmapReplacement(start, length, &result)) { | |
879 result = sys_munmap(start, length); | |
880 } | |
881 return result; | |
882 } | |
883 | |
884 #else // defined(__linux) && | |
885 // (defined(__i386__) || defined(__x86_64__) || defined(__PPC__)) | |
886 | |
887 /*static*/void* MallocHook::UnhookedMMap(void *start, size_t length, int prot, | |
888 int flags, int fd, off_t offset) { | |
889 void* result; | 707 void* result; |
890 if (!MallocHook::InvokeMmapReplacement( | 708 if (!MallocHook::InvokeMmapReplacement( |
891 start, length, prot, flags, fd, offset, &result)) { | 709 start, length, prot, flags, fd, offset, &result)) { |
892 result = mmap(start, length, prot, flags, fd, offset); | 710 result = mmap(start, length, prot, flags, fd, offset); |
893 } | 711 } |
894 return result; | 712 return result; |
895 } | 713 } |
896 | 714 |
897 /*static*/int MallocHook::UnhookedMUnmap(void *start, size_t length) { | 715 /*static*/int MallocHook::UnhookedMUnmap(void *start, size_t length) { |
898 int result; | 716 int result; |
899 if (!MallocHook::InvokeMunmapReplacement(start, length, &result)) { | 717 if (!MallocHook::InvokeMunmapReplacement(start, length, &result)) { |
900 result = munmap(start, length); | 718 result = munmap(start, length); |
901 } | 719 } |
902 return result; | 720 return result; |
903 } | 721 } |
904 | 722 |
905 #endif // defined(__linux) && | 723 #endif |
906 // (defined(__i386__) || defined(__x86_64__) || defined(__PPC__)) | |
OLD | NEW |