Changes in trunk/Source/WTF/wtf/FastMalloc.cpp [164727:164742] in webkit
- File:
-
- 1 edited
Legend:
- Unmodified
- Added
- Removed
-
trunk/Source/WTF/wtf/FastMalloc.cpp
r164727 r164742 92 92 93 93 #if OS(DARWIN) 94 #include <mach/mach_init.h> 94 95 #include <malloc/malloc.h> 95 96 #endif … … 630 631 //------------------------------------------------------------------- 631 632 633 // Type that can hold the length of a run of pages 634 typedef uintptr_t Length; 635 632 636 // Not all possible combinations of the following parameters make 633 637 // sense. In particular, if kMaxSize increases, you may have to 634 638 // increase kNumClasses as well. 635 #if OS(DARWIN) 636 # define K_PAGE_SHIFT PAGE_SHIFT 637 # if (K_PAGE_SHIFT == 12) 638 # define K_NUM_CLASSES 68 639 # elif (K_PAGE_SHIFT == 14) 640 # define K_NUM_CLASSES 77 641 # else 642 # error "Unsupported PAGE_SHIFT amount" 643 # endif 644 #else 645 # define K_PAGE_SHIFT 12 646 # define K_NUM_CLASSES 68 647 #endif 648 static const size_t kPageShift = K_PAGE_SHIFT; 649 static const size_t kPageSize = 1 << kPageShift; 639 #define K_PAGE_SHIFT_MIN 12 640 #define K_PAGE_SHIFT_MAX 14 641 #define K_NUM_CLASSES_MAX 77 642 static size_t kPageShift = 0; 643 static size_t kNumClasses = 0; 644 static size_t kPageSize = 0; 645 static Length kMaxValidPages = 0; 650 646 static const size_t kMaxSize = 32u * 1024; 651 647 static const size_t kAlignShift = 3; 652 648 static const size_t kAlignment = 1 << kAlignShift; 653 static const size_t kNumClasses = K_NUM_CLASSES;654 649 655 650 // Allocates a big block of memory for the pagemap once we reach more than … … 663 658 // have small limits on the number of mmap() regions per 664 659 // address-space. 665 static const size_t kMinSystemAlloc = 1 << (20 - kPageShift);660 static const size_t kMinSystemAlloc = 1 << (20 - K_PAGE_SHIFT_MAX); 666 661 667 662 // Number of objects to move between a per-thread list and a central … … 670 665 // it too big may temporarily cause unnecessary memory wastage in the 671 666 // per-thread free list until the scavenger cleans up the list. 672 static int num_objects_to_move[ kNumClasses];667 static int num_objects_to_move[K_NUM_CLASSES_MAX]; 673 668 674 669 // Maximum length we allow a per-thread free-list to have before we … … 766 761 767 762 // Mapping from size class to max size storable in that class 768 static size_t class_to_size[ kNumClasses];763 static size_t class_to_size[K_NUM_CLASSES_MAX]; 769 764 770 765 // Mapping from size class to number of pages to allocate at a time 771 static size_t class_to_pages[ kNumClasses];766 static size_t class_to_pages[K_NUM_CLASSES_MAX]; 772 767 773 768 // Hardened singly linked list. We make this a class to allow compiler to … … 814 809 // class is initially given one TCEntry which also means that the maximum any 815 810 // one class can have is kNumClasses. 816 static const int kNumTransferEntries = kNumClasses; 811 #define K_NUM_TRANSFER_ENTRIES_MAX static_cast<int>(K_NUM_CLASSES_MAX) 812 #define kNumTransferEntries static_cast<int>(kNumClasses) 817 813 818 814 // Note: the following only works for "n"s that fit in 32-bits, but … … 918 914 // Initialize the mapping arrays 919 915 static void InitSizeClasses() { 916 #if OS(DARWIN) 917 kPageShift = vm_page_shift; 918 switch (kPageShift) { 919 case 12: 920 kNumClasses = 68; 921 break; 922 case 14: 923 kNumClasses = 77; 924 break; 925 default: 926 CRASH(); 927 }; 928 #else 929 kPageShift = 12; 930 kNumClasses = 68; 931 #endif 932 kPageSize = 1 << kPageShift; 933 kMaxValidPages = (~static_cast<Length>(0)) >> kPageShift; 934 920 935 // Do some sanity checking on add_amount[]/shift_amount[]/class_array[] 921 936 if (ClassIndex(0) < 0) { … … 1145 1160 typedef uintptr_t PageID; 1146 1161 1147 // Type that can hold the length of a run of pages1148 typedef uintptr_t Length;1149 1150 static const Length kMaxValidPages = (~static_cast<Length>(0)) >> kPageShift;1151 1152 1162 // Convert byte size into pages. This won't overflow, but may return 1153 1163 // an unreasonably large value if bytes is huge enough. 1154 1164 static inline Length pages(size_t bytes) { 1165 ASSERT(kPageShift && kNumClasses && kPageSize); 1155 1166 return (bytes >> kPageShift) + 1156 1167 ((bytes & (kPageSize - 1)) > 0 ? 1 : 0); … … 1160 1171 // allocated 1161 1172 static size_t AllocationSize(size_t bytes) { 1173 ASSERT(kPageShift && kNumClasses && kPageSize); 1162 1174 if (bytes > kMaxSize) { 1163 1175 // Large object: we allocate an integral number of pages … … 1432 1444 // end up getting all the TCEntries quota in the system we just preallocate 1433 1445 // sufficient number of entries here. 1434 TCEntry tc_slots_[ kNumTransferEntries];1446 TCEntry tc_slots_[K_NUM_TRANSFER_ENTRIES_MAX]; 1435 1447 1436 1448 // Number of currently used cached entries in tc_slots_. This variable is … … 1654 1666 template <int BITS> class MapSelector { 1655 1667 public: 1656 typedef TCMalloc_PageMap3<BITS- kPageShift> Type;1668 typedef TCMalloc_PageMap3<BITS-K_PAGE_SHIFT_MIN> Type; 1657 1669 typedef PackedCache<BITS, uint64_t> CacheType; 1658 1670 }; … … 1672 1684 template <> class MapSelector<64> { 1673 1685 public: 1674 typedef TCMalloc_PageMap3<64 - kPageShift- kBitsUnusedOn64Bit> Type;1686 typedef TCMalloc_PageMap3<64 - K_PAGE_SHIFT_MIN - kBitsUnusedOn64Bit> Type; 1675 1687 typedef PackedCache<64, uint64_t> CacheType; 1676 1688 }; … … 1680 1692 template <> class MapSelector<32> { 1681 1693 public: 1682 typedef TCMalloc_PageMap2<32 - kPageShift> Type;1683 typedef PackedCache<32 - kPageShift, uint16_t> CacheType;1694 typedef TCMalloc_PageMap2<32 - K_PAGE_SHIFT_MIN> Type; 1695 typedef PackedCache<32 - K_PAGE_SHIFT_MIN, uint16_t> CacheType; 1684 1696 }; 1685 1697 … … 1778 1790 // Return number of free bytes in heap 1779 1791 uint64_t FreeBytes() const { 1792 ASSERT(kPageShift && kNumClasses && kPageSize); 1780 1793 return (static_cast<uint64_t>(free_pages_) << kPageShift); 1781 1794 } … … 1913 1926 void TCMalloc_PageHeap::init() 1914 1927 { 1928 ASSERT(kPageShift && kNumClasses && kPageSize); 1929 1915 1930 pagemap_.init(MetaDataAlloc); 1916 1931 pagemap_cache_ = PageMapCache(0); … … 1927 1942 // Start scavenging at kMaxPages list 1928 1943 scavenge_index_ = kMaxPages-1; 1929 COMPILE_ASSERT(kNumClasses <= (1 << PageMapCache::kValuebits), valuebits);1944 ASSERT(kNumClasses <= (1 << PageMapCache::kValuebits)); 1930 1945 DLL_Init(&large_.normal, entropy_); 1931 1946 DLL_Init(&large_.returned, entropy_); … … 2068 2083 void TCMalloc_PageHeap::scavenge() 2069 2084 { 2085 ASSERT(kPageShift && kNumClasses && kPageSize); 2070 2086 size_t pagesToRelease = min_free_committed_pages_since_last_scavenge_ * kScavengePercentage; 2071 2087 size_t targetPageCount = std::max<size_t>(kMinimumFreeCommittedPageCount, free_committed_pages_ - pagesToRelease); … … 2229 2245 2230 2246 inline void TCMalloc_PageHeap::Carve(Span* span, Length n, bool released) { 2247 ASSERT(kPageShift && kNumClasses && kPageSize); 2231 2248 ASSERT(n > 0); 2232 2249 DLL_Remove(span, entropy_); … … 2265 2282 static ALWAYS_INLINE void mergeDecommittedStates(Span* destination, Span* other) 2266 2283 { 2284 ASSERT(kPageShift && kNumClasses && kPageSize); 2267 2285 if (destination->decommitted && !other->decommitted) { 2268 2286 TCMalloc_SystemRelease(reinterpret_cast<void*>(other->start << kPageShift), … … 2368 2386 #if !USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY 2369 2387 void TCMalloc_PageHeap::IncrementalScavenge(Length n) { 2388 ASSERT(kPageShift && kNumClasses && kPageSize); 2370 2389 // Fast path; not yet time to release memory 2371 2390 scavenge_counter_ -= n; … … 2429 2448 #ifdef WTF_CHANGES 2430 2449 size_t TCMalloc_PageHeap::ReturnedBytes() const { 2450 ASSERT(kPageShift && kNumClasses && kPageSize); 2431 2451 size_t result = 0; 2432 2452 for (unsigned s = 0; s < kMaxPages; s++) { … … 2444 2464 #ifndef WTF_CHANGES 2445 2465 static double PagesToMB(uint64_t pages) { 2466 ASSERT(kPageShift && kNumClasses && kPageSize); 2446 2467 return (pages << kPageShift) / 1048576.0; 2447 2468 } … … 2510 2531 2511 2532 bool TCMalloc_PageHeap::GrowHeap(Length n) { 2533 ASSERT(kPageShift && kNumClasses && kPageSize); 2512 2534 ASSERT(kMaxPages >= kMinSystemAlloc); 2513 2535 if (n > kMaxValidPages) return false; … … 2606 2628 2607 2629 void TCMalloc_PageHeap::ReleaseFreeList(Span* list, Span* returned) { 2630 ASSERT(kPageShift && kNumClasses && kPageSize); 2608 2631 // Walk backwards through list so that when we push these 2609 2632 // spans on the "returned" list, we preserve the order. … … 2739 2762 ThreadIdentifier tid_; // Which thread owns it 2740 2763 bool in_setspecific_; // Called pthread_setspecific? 2741 FreeList list_[ kNumClasses]; // Array indexed by size-class2764 FreeList list_[K_NUM_CLASSES_MAX]; // Array indexed by size-class 2742 2765 2743 2766 // We sample allocations, biased by the size of the allocation … … 2795 2818 void enumerateFreeObjects(Finder& finder, const Reader& reader) 2796 2819 { 2820 ASSERT(kPageShift && kNumClasses && kPageSize); 2797 2821 for (unsigned sizeClass = 0; sizeClass < kNumClasses; sizeClass++) 2798 2822 list_[sizeClass].enumerateFreeObjects(finder, reader); … … 2807 2831 // Central cache -- a collection of free-lists, one per size-class. 2808 2832 // We have a separate lock per free-list to reduce contention. 2809 static TCMalloc_Central_FreeListPadded central_cache[ kNumClasses];2833 static TCMalloc_Central_FreeListPadded central_cache[K_NUM_CLASSES_MAX]; 2810 2834 2811 2835 // Page-level allocator … … 2963 2987 2964 2988 void TCMalloc_Central_FreeList::Init(size_t cl, uintptr_t entropy) { 2989 ASSERT(kPageShift && kNumClasses && kPageSize); 2965 2990 lock_.Init(); 2966 2991 size_class_ = cl; … … 2987 3012 2988 3013 ALWAYS_INLINE void TCMalloc_Central_FreeList::ReleaseToSpans(HardenedSLL object) { 3014 ASSERT(kPageShift && kNumClasses && kPageSize); 2989 3015 const PageID p = reinterpret_cast<uintptr_t>(object.value()) >> kPageShift; 2990 3016 Span* span = pageheap->GetDescriptor(p); … … 3033 3059 ALWAYS_INLINE bool TCMalloc_Central_FreeList::EvictRandomSizeClass( 3034 3060 size_t locked_size_class, bool force) { 3061 ASSERT(kPageShift && kNumClasses && kPageSize); 3035 3062 static int race_counter = 0; 3036 3063 int t = race_counter++; // Updated without a lock, but who cares. … … 3048 3075 3049 3076 bool TCMalloc_Central_FreeList::MakeCacheSpace() { 3077 ASSERT(kPageShift && kNumClasses && kPageSize); 3050 3078 // Is there room in the cache? 3051 3079 if (used_slots_ < cache_size_) return true; … … 3102 3130 3103 3131 void TCMalloc_Central_FreeList::InsertRange(HardenedSLL start, HardenedSLL end, int N) { 3132 ASSERT(kPageShift && kNumClasses && kPageSize); 3104 3133 SpinLockHolder h(&lock_); 3105 3134 if (N == num_objects_to_move[size_class_] && … … 3184 3213 // Fetch memory from the system and add to the central cache freelist. 3185 3214 ALWAYS_INLINE void TCMalloc_Central_FreeList::Populate() { 3215 ASSERT(kPageShift && kNumClasses && kPageSize); 3186 3216 // Release central list lock while operating on pageheap 3187 3217 lock_.Unlock(); … … 3270 3300 3271 3301 void TCMalloc_ThreadCache::Init(ThreadIdentifier tid, uintptr_t entropy) { 3302 ASSERT(kPageShift && kNumClasses && kPageSize); 3272 3303 size_ = 0; 3273 3304 next_ = NULL; … … 3292 3323 3293 3324 void TCMalloc_ThreadCache::Cleanup() { 3325 ASSERT(kPageShift && kNumClasses && kPageSize); 3294 3326 // Put unused memory back into central cache 3295 3327 for (size_t cl = 0; cl < kNumClasses; ++cl) { … … 3366 3398 // Release idle memory to the central cache 3367 3399 inline void TCMalloc_ThreadCache::Scavenge() { 3400 ASSERT(kPageShift && kNumClasses && kPageSize); 3368 3401 // If the low-water mark for the free list is L, it means we would 3369 3402 // not have had to allocate anything from the central cache even if … … 3658 3691 3659 3692 void TCMalloc_ThreadCache::Print() const { 3693 ASSERT(kPageShift && kNumClasses && kPageSize); 3660 3694 for (size_t cl = 0; cl < kNumClasses; ++cl) { 3661 3695 MESSAGE(" %5" PRIuS " : %4d len; %4d lo\n", … … 3679 3713 // Get stats into "r". Also get per-size-class counts if class_count != NULL 3680 3714 static void ExtractStats(TCMallocStats* r, uint64_t* class_count) { 3715 ASSERT(kPageShift && kNumClasses && kPageSize); 3681 3716 r->central_bytes = 0; 3682 3717 r->transfer_bytes = 0; … … 3716 3751 // WRITE stats to "out" 3717 3752 static void DumpStats(TCMalloc_Printer* out, int level) { 3753 ASSERT(kPageShift && kNumClasses && kPageSize); 3718 3754 TCMallocStats stats; 3719 3755 uint64_t class_count[kNumClasses]; … … 4004 4040 #if !ASSERT_DISABLED 4005 4041 static inline bool CheckCachedSizeClass(void *ptr) { 4042 ASSERT(kPageShift && kNumClasses && kPageSize); 4006 4043 PageID p = reinterpret_cast<uintptr_t>(ptr) >> kPageShift; 4007 4044 size_t cached_value = pageheap->GetSizeClassIfCached(p); … … 4018 4055 4019 4056 static inline void* SpanToMallocResult(Span *span) { 4057 ASSERT(kPageShift && kNumClasses && kPageSize); 4020 4058 ASSERT_SPAN_COMMITTED(span); 4021 4059 pageheap->CacheSizeClass(span->start, 0); … … 4071 4109 if (ptr == NULL) return; 4072 4110 ASSERT(pageheap != NULL); // Should not call free() before malloc() 4111 ASSERT(kPageShift && kNumClasses && kPageSize); 4073 4112 const PageID p = reinterpret_cast<uintptr_t>(ptr) >> kPageShift; 4074 4113 Span* span = pageheap->GetDescriptor(p); … … 4122 4161 ASSERT(align > 0); 4123 4162 if (pageheap == NULL) TCMalloc_ThreadCache::InitModule(); 4163 ASSERT(kPageShift && kNumClasses && kPageSize); 4124 4164 4125 4165 // Allocate at least one byte to avoid boundary conditions below … … 4441 4481 new_size += Internal::ValidationBufferSize; 4442 4482 #endif 4483 4484 ASSERT(pageheap != NULL); // Should not call realloc() before malloc() 4485 ASSERT(kPageShift && kNumClasses && kPageSize); 4443 4486 4444 4487 // Get the size of the old entry … … 4660 4703 FastMallocStatistics fastMallocStatistics() 4661 4704 { 4705 ASSERT(kPageShift && kNumClasses && kPageSize); 4662 4706 FastMallocStatistics statistics; 4663 4707 … … 4681 4725 size_t fastMallocSize(const void* ptr) 4682 4726 { 4727 if (pageheap == NULL) TCMalloc_ThreadCache::InitModule(); 4728 ASSERT(kPageShift && kNumClasses && kPageSize); 4729 4683 4730 #if ENABLE(WTF_MALLOC_VALIDATION) 4684 4731 return Internal::fastMallocValidationHeader(const_cast<void*>(ptr))->m_size; … … 4792 4839 int visit(void* ptr) const 4793 4840 { 4841 ASSERT(kPageShift && kNumClasses && kPageSize); 4794 4842 if (!ptr) 4795 4843 return 1; … … 4839 4887 void recordPendingRegions() 4840 4888 { 4889 ASSERT(kPageShift && kNumClasses && kPageSize); 4890 4841 4891 bool recordRegionsContainingPointers = m_typeMask & MALLOC_PTR_REGION_RANGE_TYPE; 4842 4892 bool recordAllocations = m_typeMask & MALLOC_PTR_IN_USE_RANGE_TYPE; … … 4887 4937 int visit(void* ptr) 4888 4938 { 4939 ASSERT(kPageShift && kNumClasses && kPageSize); 4889 4940 if (!ptr) 4890 4941 return 1;
Note: See TracChangeset
for help on using the changeset viewer.