⚠ Archived content — this site is no longer maintained.   Current WebKit documentation is at docs.webkit.org.

Changeset 167570 in webkit


Ignore:
Timestamp:
Apr 20, 2014, 12:33:14 PM (12 years ago)
Author:
ggaren@apple.com
Message:

bmalloc: Segregate pages by objects size
​https://bugs.webkit.org/show_bug.cgi?id=131909

Reviewed by Andreas Kling.

2% reduction in memory-at-end on the Membuster memory_warning benchmarks.

  • bmalloc/Allocator.cpp:

(bmalloc::Allocator::allocateSlowCase):

  • bmalloc/Allocator.h:

(bmalloc::Allocator::allocateFastCase):
(bmalloc::Allocator::smallAllocatorFor): Use the new shared helper
function for size class calculation.

  • bmalloc/Deallocator.cpp:

(bmalloc::Deallocator::Deallocator):
(bmalloc::Deallocator::scavenge):
(bmalloc::Deallocator::deallocateSmallLine):
(bmalloc::Deallocator::allocateSmallLine):

  • bmalloc/Deallocator.h: Keep a cache for every size class, since the

cache can't be shared anymore.

  • bmalloc/Heap.cpp:

(bmalloc::Heap::allocateSmallLineSlowCase):

  • bmalloc/Heap.h:

(bmalloc::Heap::deallocateSmallLine): Ditto.

(bmalloc::Heap::allocateSmallLine): Check size class in addition to
page refcount when allocating a line because we might have deallocated
the page and the recycled it for another size class.

(bmalloc::Heap::deallocateMediumLine):
(bmalloc::Heap::allocateMediumLine):

  • bmalloc/Line.h:

(bmalloc::Line::refCount):

  • bmalloc/Page.h:

(bmalloc::Page::refCount):
(bmalloc::Page::smallSizeClass):
(bmalloc::Page::setSmallSizeClass):
(bmalloc::Page<Traits>::refCount): Deleted.

  • bmalloc/Sizes.h:

(bmalloc::Sizes::smallSizeClassFor): New shared API for computing
an index into an array from a size.

Location:
trunk/Source/bmalloc
Files:
10 edited

Legend:

Unmodified
Added
Removed

  • trunk/Source/bmalloc/ChangeLog

    r167554 r167570  
     12014-04-20  Geoffrey Garen  <ggaren@apple.com>
     2
     3        bmalloc: Segregate pages by objects size
     4        https://bugs.webkit.org/show_bug.cgi?id=131909
     5
     6        Reviewed by Andreas Kling.
     7
     8        2% reduction in memory-at-end on the Membuster memory_warning benchmarks.
     9
     10        * bmalloc/Allocator.cpp:
     11        (bmalloc::Allocator::allocateSlowCase):
     12        * bmalloc/Allocator.h:
     13        (bmalloc::Allocator::allocateFastCase):
     14        (bmalloc::Allocator::smallAllocatorFor): Use the new shared helper
     15        function for size class calculation.
     16
     17        * bmalloc/Deallocator.cpp:
     18        (bmalloc::Deallocator::Deallocator):
     19        (bmalloc::Deallocator::scavenge):
     20        (bmalloc::Deallocator::deallocateSmallLine):
     21        (bmalloc::Deallocator::allocateSmallLine):
     22        * bmalloc/Deallocator.h: Keep a cache for every size class, since the
     23        cache can't be shared anymore.
     24
     25        * bmalloc/Heap.cpp:
     26        (bmalloc::Heap::allocateSmallLineSlowCase):
     27        * bmalloc/Heap.h:
     28        (bmalloc::Heap::deallocateSmallLine): Ditto.
     29
     30        (bmalloc::Heap::allocateSmallLine): Check size class in addition to
     31        page refcount when allocating a line because we might have deallocated
     32        the page and the recycled it for another size class.
     33
     34        (bmalloc::Heap::deallocateMediumLine):
     35        (bmalloc::Heap::allocateMediumLine):
     36        * bmalloc/Line.h:
     37        (bmalloc::Line::refCount):
     38        * bmalloc/Page.h:
     39        (bmalloc::Page::refCount):
     40        (bmalloc::Page::smallSizeClass):
     41        (bmalloc::Page::setSmallSizeClass):
     42        (bmalloc::Page<Traits>::refCount): Deleted.
     43        * bmalloc/Sizes.h:
     44        (bmalloc::Sizes::smallSizeClassFor): New shared API for computing
     45        an index into an array from a size.
     46
    1472014-04-19  Geoffrey Garen  <ggaren@apple.com>
    248

  • trunk/Source/bmalloc/bmalloc/Allocator.cpp

    r167546 r167570  
    146146)
    147147    if (size <= smallMax) {
    148         SmallAllocator& allocator = smallAllocatorFor(size);
     148        size_t smallSizeClass = smallSizeClassFor(size);
     149        SmallAllocator& allocator = m_smallAllocators[smallSizeClass];
    149150        log(allocator);
    150         allocator.refill(m_deallocator.allocateSmallLine());
     151        allocator.refill(m_deallocator.allocateSmallLine(smallSizeClass));
    151152        return allocator.allocate();
    152153    }

  • trunk/Source/bmalloc/bmalloc/Allocator.h

    r167546 r167570  
    5151
    5252private:
    53     SmallAllocator& smallAllocatorFor(size_t);
    5453    void* allocateFastCase(SmallAllocator&);
    5554
    … …  
    7372};
    7473
    75 inline SmallAllocator& Allocator::smallAllocatorFor(size_t size)
    76 {
    77     size_t index = mask((size - 1ul) / alignment, m_smallAllocators.size() - 1);
    78     return m_smallAllocators[index];
    79 }
    80 
    8174inline bool Allocator::allocateFastCase(size_t size, void*& object)
    8275{
    … …  
    8477        return false;
    8578
    86     SmallAllocator& allocator = smallAllocatorFor(size);
     79    SmallAllocator& allocator = m_smallAllocators[smallSizeClassFor(size)];
    8780    if (!allocator.canAllocate())
    8881        return false;

  • trunk/Source/bmalloc/bmalloc/Deallocator.cpp

    r167546 r167570  
    4141Deallocator::Deallocator()
    4242    : m_objectLog()
    43     , m_smallLineCache()
     43    , m_smallLineCaches()
    4444    , m_mediumLineCache()
    4545{
    … …  
    5858    Heap* heap = PerProcess<Heap>::getFastCase();
    5959   
    60     while (m_smallLineCache.size())
    61         heap->deallocateSmallLine(lock, m_smallLineCache.pop());
     60    for (auto& smallLineCache : m_smallLineCaches) {
     61        while (smallLineCache.size())
     62            heap->deallocateSmallLine(lock, smallLineCache.pop());
     63    }
    6264    while (m_mediumLineCache.size())
    6365        heap->deallocateMediumLine(lock, m_mediumLineCache.pop());
    … …  
    120122void Deallocator::deallocateSmallLine(std::lock_guard<StaticMutex>& lock, SmallLine* line)
    121123{
    122     if (m_smallLineCache.size() == m_smallLineCache.capacity())
     124    SmallLineCache& smallLineCache = m_smallLineCaches[SmallPage::get(line)->smallSizeClass()];
     125    if (smallLineCache.size() == smallLineCache.capacity())
    123126        return PerProcess<Heap>::getFastCase()->deallocateSmallLine(lock, line);
    124127
    125     m_smallLineCache.push(line);
     128    smallLineCache.push(line);
    126129}
    127130
    128 SmallLine* Deallocator::allocateSmallLine()
     131SmallLine* Deallocator::allocateSmallLine(size_t smallSizeClass)
    129132{
    130     if (!m_smallLineCache.size()) {
     133    SmallLineCache& smallLineCache = m_smallLineCaches[smallSizeClass];
     134    if (!smallLineCache.size()) {
    131135        std::lock_guard<StaticMutex> lock(PerProcess<Heap>::mutex());
    132136        Heap* heap = PerProcess<Heap>::getFastCase();
    133137
    134         while (m_smallLineCache.size() != m_smallLineCache.capacity())
    135             m_smallLineCache.push(heap->allocateSmallLine(lock));
     138        while (smallLineCache.size() != smallLineCache.capacity())
     139            smallLineCache.push(heap->allocateSmallLine(lock, smallSizeClass));
    136140    }
    137141
    138     return m_smallLineCache.pop();
     142    return smallLineCache.pop();
    139143}
    140144

  • trunk/Source/bmalloc/bmalloc/Deallocator.h

    r167546 r167570  
    4646
    4747    void deallocateSmallLine(std::lock_guard<StaticMutex>&, SmallLine*);
    48     SmallLine* allocateSmallLine();
     48    SmallLine* allocateSmallLine(size_t smallSizeClass);
    4949
    5050    void deallocateMediumLine(std::lock_guard<StaticMutex>&, MediumLine*);
    … …  
    5454   
    5555private:
     56    typedef FixedVector<SmallLine*, smallLineCacheCapacity> SmallLineCache;
     57    typedef FixedVector<MediumLine*, mediumLineCacheCapacity> MediumLineCache;
     58
    5659    void deallocateLarge(void*);
    5760    void deallocateXLarge(void*);
    … …  
    5962
    6063    FixedVector<void*, deallocatorLogCapacity> m_objectLog;
    61     FixedVector<SmallLine*, smallLineCacheCapacity> m_smallLineCache;
    62     FixedVector<MediumLine*, mediumLineCacheCapacity> m_mediumLineCache;
     64    std::array<SmallLineCache, smallMax / alignment> m_smallLineCaches;
     65    MediumLineCache m_mediumLineCache;
    6366};
    6467

  • trunk/Source/bmalloc/bmalloc/Heap.cpp

    r167546 r167570  
    117117}
    118118
    119 SmallLine* Heap::allocateSmallLineSlowCase(std::lock_guard<StaticMutex>& lock)
     119SmallLine* Heap::allocateSmallLineSlowCase(std::lock_guard<StaticMutex>& lock, size_t smallSizeClass)
    120120{
    121121    m_isAllocatingPages = true;
    … …  
    131131
    132132    SmallLine* line = page->begin();
     133    Vector<SmallLine*>& smallLines = m_smallLines[smallSizeClass];
    133134    for (auto it = line + 1; it != page->end(); ++it)
    134         m_smallLines.push(it);
    135 
     135        smallLines.push(it);
     136
     137    BASSERT(!line->refCount(lock));
     138    page->setSmallSizeClass(smallSizeClass);
    136139    page->ref(lock);
    137140    return line;

  • trunk/Source/bmalloc/bmalloc/Heap.h

    r167546 r167570  
    5050    Heap(std::lock_guard<StaticMutex>&);
    5151
    52     SmallLine* allocateSmallLine(std::lock_guard<StaticMutex>&);
     52    SmallLine* allocateSmallLine(std::lock_guard<StaticMutex>&, size_t smallSizeClass);
    5353    void deallocateSmallLine(std::lock_guard<StaticMutex>&, SmallLine*);
    5454
    … …  
    6767    ~Heap() = delete;
    6868
    69     SmallLine* allocateSmallLineSlowCase(std::lock_guard<StaticMutex>&);
     69    SmallLine* allocateSmallLineSlowCase(std::lock_guard<StaticMutex>&, size_t smallSizeClass);
    7070    MediumLine* allocateMediumLineSlowCase(std::lock_guard<StaticMutex>&);
    7171
    … …  
    8383    void scavengeLargeRanges(std::unique_lock<StaticMutex>&, std::chrono::milliseconds);
    8484
    85     Vector<SmallLine*> m_smallLines;
     85    std::array<Vector<SmallLine*>, smallMax / alignment> m_smallLines;
    8686    Vector<MediumLine*> m_mediumLines;
    8787
    … …  
    9999inline void Heap::deallocateSmallLine(std::lock_guard<StaticMutex>& lock, SmallLine* line)
    100100{
     101    BASSERT(!line->refCount(lock));
    101102    SmallPage* page = SmallPage::get(line);
    102103    if (page->deref(lock)) {
    … …  
    105106        return;
    106107    }
    107     m_smallLines.push(line);
     108    m_smallLines[page->smallSizeClass()].push(line);
    108109}
    109110
    110 inline SmallLine* Heap::allocateSmallLine(std::lock_guard<StaticMutex>& lock)
     111inline SmallLine* Heap::allocateSmallLine(std::lock_guard<StaticMutex>& lock, size_t smallSizeClass)
    111112{
    112     while (m_smallLines.size()) {
    113         SmallLine* line = m_smallLines.pop();
     113    Vector<SmallLine*>& smallLines = m_smallLines[smallSizeClass];
     114    while (smallLines.size()) {
     115        SmallLine* line = smallLines.pop();
    114116        SmallPage* page = SmallPage::get(line);
    115         if (!page->refCount(lock)) // The line was promoted to the small pages list.
     117        if (!page->refCount(lock) || page->smallSizeClass() != smallSizeClass) // The line was promoted to the small pages list.
    116118            continue;
     119        BASSERT(!line->refCount(lock));
    117120        page->ref(lock);
    118121        return line;
    119122    }
    120123
    121     return allocateSmallLineSlowCase(lock);
     124    return allocateSmallLineSlowCase(lock, smallSizeClass);
    122125}
    123126
    124127inline void Heap::deallocateMediumLine(std::lock_guard<StaticMutex>& lock, MediumLine* line)
    125128{
     129    BASSERT(!line->refCount(lock));
    126130    MediumPage* page = MediumPage::get(line);
    127131    if (page->deref(lock)) {
    … …  
    140144        if (!page->refCount(lock)) // The line was promoted to the medium pages list.
    141145            continue;
     146        BASSERT(!line->refCount(lock));
    142147        page->ref(lock);
    143148        return line;

  • trunk/Source/bmalloc/bmalloc/Line.h

    r167546 r167570  
    4848    void concurrentRef(unsigned char = 1);
    4949    bool deref(std::lock_guard<StaticMutex>&, unsigned char = 1);
     50    unsigned refCount(std::lock_guard<StaticMutex>&) { return m_refCount; }
    5051   
    5152    char* begin();

  • trunk/Source/bmalloc/bmalloc/Page.h

    r167540 r167570  
    4848    void ref(std::lock_guard<StaticMutex>&);
    4949    bool deref(std::lock_guard<StaticMutex>&);
    50     unsigned refCount(std::lock_guard<StaticMutex>&);
     50    unsigned refCount(std::lock_guard<StaticMutex>&) { return m_refCount; }
     51   
     52    size_t smallSizeClass() { return m_smallSizeClass; }
     53    void setSmallSizeClass(size_t smallSizeClass) { m_smallSizeClass = smallSizeClass; }
    5154   
    5255    Line* begin();
    … …  
    5558private:
    5659    unsigned char m_refCount;
     60    unsigned char m_smallSizeClass;
    5761};
    5862
    … …  
    7074    --m_refCount;
    7175    return !m_refCount;
    72 }
    73 
    74 template<typename Traits>
    75 inline unsigned Page<Traits>::refCount(std::lock_guard<StaticMutex>&)
    76 {
    77     return m_refCount;
    7876}
    7977

  • trunk/Source/bmalloc/bmalloc/Sizes.h

    r167546 r167570  
    9090   
    9191    static const std::chrono::milliseconds scavengeSleepDuration = std::chrono::milliseconds(512);
     92
     93    inline size_t smallSizeClassFor(size_t size)
     94    {
     95        static const size_t smallSizeClassMask = (smallMax / alignment) - 1;
     96        return mask((size - 1ul) / alignment, smallSizeClassMask);
     97    }
    9298};
    9399
Note: See TracChangeset for help on using the changeset viewer.