Changeset 255611 in webkit
- Timestamp:
- Feb 3, 2020 3:48:29 PM (4 years ago)
- Location:
- trunk
- Files:
-
- 5 edited
Legend:
- Unmodified
- Added
- Removed
-
trunk/Source/WTF/ChangeLog
r255577 r255611 1 2020-02-03 Alex Christensen <achristensen@webkit.org> 2 3 Reduce size of HashMap and HashSet 4 https://bugs.webkit.org/show_bug.cgi?id=207138 5 6 Reviewed by Yusuke Suzuki. 7 8 This reduces sizeof(HashMap) and sizeof(HashSet) from 24 to 8 on 64-bit systems. 9 I measured that the overwhelming majority of HashMaps and HashSets never see more than 0 elements, 10 so I moved the table metadata (tableSize, tableSizeMask, keyCount, deletedCount) to inside the 11 dynamically allocated memory. This makes another branch in size() for non-empty tables 12 and an additional read and write when rehashing in exchange for fewer writes in the constructor 13 and increased cache locality of everything that uses HashMap and HashSet, which is basically everything. 14 15 * wtf/HashTable.h: 16 (WTF::HashTable::~HashTable): 17 (WTF::HashTable::end): 18 (WTF::HashTable::end const): 19 (WTF::HashTable::random): 20 (WTF::HashTable::size const): 21 (WTF::HashTable::capacity const): 22 (WTF::HashTable::isEmpty const): 23 (WTF::HashTable::reserveInitialCapacity): 24 (WTF::HashTable::shouldExpand const): 25 (WTF::HashTable::mustRehashInPlace const): 26 (WTF::HashTable::shouldShrink const): 27 (WTF::HashTable::shrink): 28 (WTF::HashTable::makeIterator): 29 (WTF::HashTable::makeConstIterator const): 30 (WTF::HashTable::makeKnownGoodIterator): 31 (WTF::HashTable::makeKnownGoodConstIterator const): 32 (WTF::HashTable::tableSize const): 33 (WTF::HashTable::setTableSize const): 34 (WTF::HashTable::tableSizeMask const): 35 (WTF::HashTable::setTableSizeMask): 36 (WTF::HashTable::keyCount const): 37 (WTF::HashTable::setKeyCount const): 38 (WTF::HashTable::deletedCount const): 39 (WTF::HashTable::setDeletedCount const): 40 (WTF::KeyTraits>::HashTable): 41 (WTF::KeyTraits>::inlineLookup): 42 (WTF::KeyTraits>::lookupForWriting): 43 (WTF::KeyTraits>::fullLookupForWriting): 44 (WTF::KeyTraits>::addUniqueForInitialization): 45 (WTF::KeyTraits>::add): 46 (WTF::KeyTraits>::addPassingHashCode): 47 (WTF::KeyTraits>::remove): 48 (WTF::KeyTraits>::removeIf): 49 (WTF::KeyTraits>::allocateTable): 50 (WTF::KeyTraits>::deallocateTable): 51 (WTF::KeyTraits>::expand): 52 (WTF::KeyTraits>::shrinkToBestSize): 53 (WTF::KeyTraits>::deleteReleasedWeakBuckets): 54 (WTF::KeyTraits>::rehash): 55 (WTF::KeyTraits>::clear): 56 (WTF::KeyTraits>::swap): 57 (WTF::KeyTraits>::checkTableConsistencyExceptSize const): 58 1 59 2020-02-03 Sam Weinig <weinig@apple.com> 2 60 -
trunk/Source/WTF/wtf/HashTable.h
r254087 r255611 367 367 invalidateIterators(); 368 368 if (m_table) 369 deallocateTable(m_table , m_tableSize);369 deallocateTable(m_table); 370 370 #if CHECK_HASHTABLE_USE_AFTER_DESTRUCTION 371 371 m_table = (ValueType*)(uintptr_t)0xbbadbeef; … … 384 384 // buckets, and iterating an empty table is a common case that's worth optimizing. 385 385 iterator begin() { return isEmpty() ? end() : makeIterator(m_table); } 386 iterator end() { return makeKnownGoodIterator(m_table + m_tableSize); }386 iterator end() { return makeKnownGoodIterator(m_table + tableSize()); } 387 387 const_iterator begin() const { return isEmpty() ? end() : makeConstIterator(m_table); } 388 const_iterator end() const { return makeKnownGoodConstIterator(m_table + m_tableSize); }388 const_iterator end() const { return makeKnownGoodConstIterator(m_table + tableSize()); } 389 389 390 390 iterator random() … … 394 394 395 395 while (1) { 396 auto& bucket = m_table[weakRandomUint32() & m_tableSizeMask];396 auto& bucket = m_table[weakRandomUint32() & tableSizeMask()]; 397 397 if (!isEmptyOrDeletedBucket(bucket)) 398 398 return makeKnownGoodIterator(&bucket); … … 402 402 const_iterator random() const { return static_cast<const_iterator>(const_cast<HashTable*>(this)->random()); } 403 403 404 unsigned size() const { return m_keyCount; }405 unsigned capacity() const { return m_tableSize; }406 bool isEmpty() const { return ! m_keyCount; }404 unsigned size() const { return keyCount(); } 405 unsigned capacity() const { return tableSize(); } 406 bool isEmpty() const { return !keyCount(); } 407 407 408 408 void reserveInitialCapacity(unsigned keyCount) 409 409 { 410 410 ASSERT(!m_table); 411 ASSERT(!m_tableSize); 412 ASSERT(!m_deletedCount); 411 ASSERT(!tableSize()); 413 412 414 413 unsigned minimumTableSize = KeyTraits::minimumTableSize; 415 414 unsigned newTableSize = std::max(minimumTableSize, computeBestTableSize(keyCount)); 416 415 417 m_tableSize = newTableSize;418 m_tableSizeMask = newTableSize - 1;419 416 m_table = allocateTable(newTableSize); 417 setTableSize(newTableSize); 418 setTableSizeMask(newTableSize - 1); 419 setDeletedCount(0); 420 setKeyCount(0); 420 421 } 421 422 … … 469 470 private: 470 471 static ValueType* allocateTable(unsigned size); 471 static void deallocateTable(ValueType* table , unsigned size);472 static void deallocateTable(ValueType* table); 472 473 473 474 typedef std::pair<ValueType*, bool> LookupType; … … 487 488 488 489 static constexpr unsigned computeBestTableSize(unsigned keyCount); 489 bool shouldExpand() const { return ( m_keyCount + m_deletedCount) * m_maxLoad >= m_tableSize; }490 bool mustRehashInPlace() const { return m_keyCount * m_minLoad < m_tableSize* 2; }491 bool shouldShrink() const { return m_keyCount * m_minLoad < m_tableSize && m_tableSize> KeyTraits::minimumTableSize; }490 bool shouldExpand() const { return (keyCount() + deletedCount()) * maxLoad >= tableSize(); } 491 bool mustRehashInPlace() const { return keyCount() * minLoad < tableSize() * 2; } 492 bool shouldShrink() const { return keyCount() * minLoad < tableSize() && tableSize() > KeyTraits::minimumTableSize; } 492 493 ValueType* expand(ValueType* entry = nullptr); 493 void shrink() { rehash( m_tableSize/ 2, nullptr); }494 void shrink() { rehash(tableSize() / 2, nullptr); } 494 495 void shrinkToBestSize(); 495 496 … … 505 506 { return FullLookupType(LookupType(position, found), hash); } 506 507 507 iterator makeIterator(ValueType* pos) { return iterator(this, pos, m_table + m_tableSize); }508 const_iterator makeConstIterator(ValueType* pos) const { return const_iterator(this, pos, m_table + m_tableSize); }509 iterator makeKnownGoodIterator(ValueType* pos) { return iterator(this, pos, m_table + m_tableSize, HashItemKnownGood); }510 const_iterator makeKnownGoodConstIterator(ValueType* pos) const { return const_iterator(this, pos, m_table + m_tableSize, HashItemKnownGood); }508 iterator makeIterator(ValueType* pos) { return iterator(this, pos, m_table + tableSize()); } 509 const_iterator makeConstIterator(ValueType* pos) const { return const_iterator(this, pos, m_table + tableSize()); } 510 iterator makeKnownGoodIterator(ValueType* pos) { return iterator(this, pos, m_table + tableSize(), HashItemKnownGood); } 511 const_iterator makeKnownGoodConstIterator(ValueType* pos) const { return const_iterator(this, pos, m_table + tableSize(), HashItemKnownGood); } 511 512 512 513 #if ASSERT_ENABLED … … 522 523 #endif 523 524 524 static constexpr unsigned m_maxLoad = 2; 525 static constexpr unsigned m_minLoad = 6; 526 527 ValueType* m_table; 528 unsigned m_tableSize; 529 unsigned m_tableSizeMask; 530 unsigned m_keyCount; 531 unsigned m_deletedCount; 525 static constexpr unsigned maxLoad = 2; 526 static constexpr unsigned minLoad = 6; 527 528 static constexpr int tableSizeOffset = -1; 529 static constexpr int tableSizeMaskOffset = -2; 530 static constexpr int keyCountOffset = -3; 531 static constexpr int deletedCountOffset = -4; 532 static constexpr unsigned metadataSize = 4 * sizeof(unsigned); 533 534 unsigned tableSize() const { return m_table ? reinterpret_cast<unsigned*>(m_table)[tableSizeOffset] : 0; } 535 void setTableSize(unsigned size) const { ASSERT(m_table); reinterpret_cast<unsigned*>(m_table)[tableSizeOffset] = size; } 536 unsigned tableSizeMask() const { ASSERT(m_table); return m_table ? reinterpret_cast<unsigned*>(m_table)[tableSizeMaskOffset] : 0; } 537 void setTableSizeMask(unsigned mask) { ASSERT(m_table); reinterpret_cast<unsigned*>(m_table)[tableSizeMaskOffset] = mask; } 538 unsigned keyCount() const { return m_table ? reinterpret_cast<unsigned*>(m_table)[keyCountOffset] : 0; } 539 void setKeyCount(unsigned count) const { ASSERT(m_table); reinterpret_cast<unsigned*>(m_table)[keyCountOffset] = count; } 540 unsigned deletedCount() const { ASSERT(m_table); return reinterpret_cast<unsigned*>(m_table)[deletedCountOffset]; } 541 void setDeletedCount(unsigned count) const { ASSERT(m_table); reinterpret_cast<unsigned*>(m_table)[deletedCountOffset] = count; } 542 543 ValueType* m_table { nullptr }; 532 544 533 545 #if CHECK_HASHTABLE_ITERATORS … … 585 597 template<typename Key, typename Value, typename Extractor, typename HashFunctions, typename Traits, typename KeyTraits> 586 598 inline HashTable<Key, Value, Extractor, HashFunctions, Traits, KeyTraits>::HashTable() 587 : m_table(0) 588 , m_tableSize(0) 589 , m_tableSizeMask(0) 590 , m_keyCount(0) 591 , m_deletedCount(0) 599 : m_table(nullptr) 592 600 #if CHECK_HASHTABLE_ITERATORS 593 601 , m_iterators(0) … … 650 658 651 659 unsigned k = 0; 652 unsigned sizeMask = m_tableSizeMask;653 660 ValueType* table = m_table; 661 if (!table) 662 return nullptr; 663 664 unsigned sizeMask = tableSizeMask(); 654 665 unsigned h = HashTranslator::hash(key); 655 666 unsigned i = h & sizeMask; 656 657 if (!table)658 return 0;659 667 660 668 #if DUMP_HASHTABLE_STATS … … 708 716 unsigned k = 0; 709 717 ValueType* table = m_table; 710 unsigned sizeMask = m_tableSizeMask;718 unsigned sizeMask = tableSizeMask(); 711 719 unsigned h = HashTranslator::hash(key); 712 720 unsigned i = h & sizeMask; … … 769 777 unsigned k = 0; 770 778 ValueType* table = m_table; 771 unsigned sizeMask = m_tableSizeMask;779 unsigned sizeMask = tableSizeMask(); 772 780 unsigned h = HashTranslator::hash(key); 773 781 unsigned i = h & sizeMask; … … 835 843 unsigned k = 0; 836 844 ValueType* table = m_table; 837 unsigned sizeMask = m_tableSizeMask;845 unsigned sizeMask = tableSizeMask(); 838 846 unsigned h = HashTranslator::hash(key); 839 847 unsigned i = h & sizeMask; … … 916 924 unsigned k = 0; 917 925 ValueType* table = m_table; 918 unsigned sizeMask = m_tableSizeMask;926 unsigned sizeMask = tableSizeMask(); 919 927 unsigned h = HashTranslator::hash(key); 920 928 unsigned i = h & sizeMask; … … 970 978 initializeBucket(*deletedEntry); 971 979 entry = deletedEntry; 972 --m_deletedCount;980 setDeletedCount(deletedCount() - 1); 973 981 } 974 982 975 983 HashTranslator::translate(*entry, std::forward<T>(key), std::forward<Extra>(extra)); 976 ++m_keyCount;984 setKeyCount(keyCount() + 1); 977 985 978 986 if (shouldExpand()) … … 1008 1016 if (isDeletedBucket(*entry)) { 1009 1017 initializeBucket(*entry); 1010 --m_deletedCount;1018 setDeletedCount(deletedCount() - 1); 1011 1019 } 1012 1020 1013 1021 HashTranslator::translate(*entry, std::forward<T>(key), std::forward<Extra>(extra), h); 1014 ++m_keyCount;1022 setKeyCount(keyCount() + 1); 1015 1023 1016 1024 if (shouldExpand()) … … 1106 1114 1107 1115 deleteBucket(*pos); 1108 ++m_deletedCount;1109 --m_keyCount;1116 setDeletedCount(deletedCount() + 1); 1117 setKeyCount(keyCount() - 1); 1110 1118 1111 1119 if (shouldShrink()) … … 1158 1166 ValueType* table = m_table; 1159 1167 1160 for (unsigned i = m_tableSize; i--;) {1168 for (unsigned i = tableSize(); i--;) { 1161 1169 ValueType& bucket = table[i]; 1162 1170 if (isEmptyOrDeletedBucket(bucket)) … … 1169 1177 ++removedBucketCount; 1170 1178 } 1171 m_deletedCount += removedBucketCount; 1172 m_keyCount -= removedBucketCount; 1179 if (removedBucketCount) { 1180 setDeletedCount(deletedCount() + removedBucketCount); 1181 setKeyCount(keyCount() - removedBucketCount); 1182 } 1173 1183 1174 1184 if (shouldShrink()) … … 1182 1192 auto HashTable<Key, Value, Extractor, HashFunctions, Traits, KeyTraits>::allocateTable(unsigned size) -> ValueType* 1183 1193 { 1194 static_assert(!(metadataSize % alignof(ValueType))); 1195 1184 1196 // would use a template member function with explicit specializations here, but 1185 1197 // gcc doesn't appear to support that 1186 1198 if (Traits::emptyValueIsZero) 1187 return static_cast<ValueType*>(HashTableMalloc::zeroedMalloc(size * sizeof(ValueType)));1188 1189 ValueType* result = static_cast<ValueType*>(HashTableMalloc::malloc(size * sizeof(ValueType)));1199 return reinterpret_cast<ValueType*>(static_cast<char*>(HashTableMalloc::zeroedMalloc(metadataSize + size * sizeof(ValueType))) + metadataSize); 1200 1201 ValueType* result = reinterpret_cast<ValueType*>(static_cast<char*>(HashTableMalloc::malloc(metadataSize + size * sizeof(ValueType))) + metadataSize); 1190 1202 for (unsigned i = 0; i < size; i++) 1191 1203 initializeBucket(result[i]); … … 1194 1206 1195 1207 template<typename Key, typename Value, typename Extractor, typename HashFunctions, typename Traits, typename KeyTraits> 1196 void HashTable<Key, Value, Extractor, HashFunctions, Traits, KeyTraits>::deallocateTable(ValueType* table, unsigned size) 1197 { 1208 void HashTable<Key, Value, Extractor, HashFunctions, Traits, KeyTraits>::deallocateTable(ValueType* table) 1209 { 1210 unsigned size = reinterpret_cast<unsigned*>(table)[tableSizeOffset]; 1198 1211 for (unsigned i = 0; i < size; ++i) { 1199 1212 if (!isDeletedBucket(table[i])) 1200 1213 table[i].~ValueType(); 1201 1214 } 1202 HashTableMalloc::free( table);1215 HashTableMalloc::free(reinterpret_cast<char*>(table) - metadataSize); 1203 1216 } 1204 1217 … … 1210 1223 1211 1224 unsigned newSize; 1212 if (m_tableSize == 0) 1225 unsigned oldSize = tableSize(); 1226 if (!oldSize) 1213 1227 newSize = KeyTraits::minimumTableSize; 1214 1228 else if (mustRehashInPlace()) 1215 newSize = m_tableSize;1229 newSize = oldSize; 1216 1230 else 1217 newSize = m_tableSize * 2;1231 newSize = oldSize * 2; 1218 1232 1219 1233 return rehash(newSize, entry); … … 1240 1254 { 1241 1255 unsigned minimumTableSize = KeyTraits::minimumTableSize; 1242 rehash(std::max(minimumTableSize, computeBestTableSize( m_keyCount)), nullptr);1256 rehash(std::max(minimumTableSize, computeBestTableSize(keyCount())), nullptr); 1243 1257 } 1244 1258 … … 1246 1260 void HashTable<Key, Value, Extractor, HashFunctions, Traits, KeyTraits>::deleteReleasedWeakBuckets() 1247 1261 { 1248 for (unsigned i = 0; i < m_tableSize; ++i) { 1262 unsigned tableSize = this->tableSize(); 1263 for (unsigned i = 0; i < tableSize; ++i) { 1249 1264 auto& entry = m_table[i]; 1250 1265 if (isReleasedWeakBucket(entry)) { 1251 1266 deleteBucket(entry); 1252 ++m_deletedCount;1253 --m_keyCount;1267 setDeletedCount(deletedCount() + 1); 1268 setKeyCount(keyCount() - 1); 1254 1269 } 1255 1270 } … … 1261 1276 internalCheckTableConsistencyExceptSize(); 1262 1277 1263 unsigned oldTableSize = m_tableSize;1278 unsigned oldTableSize = tableSize(); 1264 1279 ValueType* oldTable = m_table; 1265 1280 … … 1274 1289 #endif 1275 1290 1276 m_tableSize = newTableSize; 1277 m_tableSizeMask = newTableSize - 1; 1291 unsigned oldKeyCount = keyCount(); 1278 1292 m_table = allocateTable(newTableSize); 1293 setTableSize(newTableSize); 1294 setTableSizeMask(newTableSize - 1); 1295 setDeletedCount(0); 1296 setKeyCount(oldKeyCount); 1279 1297 1280 1298 Value* newEntry = nullptr; … … 1295 1313 ASSERT(std::addressof(oldEntry) != entry); 1296 1314 oldEntry.~ValueType(); 1297 --m_keyCount;1315 setKeyCount(keyCount() - 1); 1298 1316 continue; 1299 1317 } … … 1307 1325 } 1308 1326 1309 m_deletedCount = 0; 1310 1311 HashTableMalloc::free(oldTable); 1327 if (oldTable) 1328 HashTableMalloc::free(reinterpret_cast<char*>(oldTable) - metadataSize); 1312 1329 1313 1330 internalCheckTableConsistency(); … … 1322 1339 return; 1323 1340 1324 deallocateTable(m_table, m_tableSize); 1325 m_table = 0; 1326 m_tableSize = 0; 1327 m_tableSizeMask = 0; 1328 m_keyCount = 0; 1329 m_deletedCount = 0; 1341 deallocateTable(m_table); 1342 m_table = nullptr; 1330 1343 } 1331 1344 … … 1333 1346 HashTable<Key, Value, Extractor, HashFunctions, Traits, KeyTraits>::HashTable(const HashTable& other) 1334 1347 : m_table(nullptr) 1335 , m_tableSize(0)1336 , m_tableSizeMask(0)1337 , m_keyCount(0)1338 , m_deletedCount(0)1339 1348 #if CHECK_HASHTABLE_ITERATORS 1340 1349 , m_iterators(nullptr) … … 1349 1358 return; 1350 1359 1351 m_tableSize = computeBestTableSize(otherKeyCount); 1352 m_tableSizeMask = m_tableSize - 1; 1353 m_keyCount = otherKeyCount; 1354 m_table = allocateTable(m_tableSize); 1360 unsigned bestTableSize = computeBestTableSize(otherKeyCount); 1361 m_table = allocateTable(bestTableSize); 1362 setTableSize(bestTableSize); 1363 setTableSizeMask(bestTableSize - 1); 1364 setKeyCount(otherKeyCount); 1365 setDeletedCount(0); 1355 1366 1356 1367 for (const auto& otherValue : other) … … 1365 1376 1366 1377 std::swap(m_table, other.m_table); 1367 std::swap(m_tableSize, other.m_tableSize);1368 std::swap(m_tableSizeMask, other.m_tableSizeMask);1369 std::swap(m_keyCount, other.m_keyCount);1370 std::swap(m_deletedCount, other.m_deletedCount);1371 1378 1372 1379 #if DUMP_HASHTABLE_STATS_PER_TABLE … … 1392 1399 other.invalidateIterators(); 1393 1400 1394 m_table = other.m_table; 1395 m_tableSize = other.m_tableSize; 1396 m_tableSizeMask = other.m_tableSizeMask; 1397 m_keyCount = other.m_keyCount; 1398 m_deletedCount = other.m_deletedCount; 1399 1400 other.m_table = nullptr; 1401 other.m_tableSize = 0; 1402 other.m_tableSizeMask = 0; 1403 other.m_keyCount = 0; 1404 other.m_deletedCount = 0; 1401 m_table = std::exchange(other.m_table, nullptr); 1405 1402 1406 1403 #if DUMP_HASHTABLE_STATS_PER_TABLE … … 1436 1433 unsigned count = 0; 1437 1434 unsigned deletedCount = 0; 1438 for (unsigned j = 0; j < m_tableSize; ++j) { 1435 unsigned tableSize = this->tableSize(); 1436 for (unsigned j = 0; j < tableSize; ++j) { 1439 1437 ValueType* entry = m_table + j; 1440 1438 if (isEmptyBucket(*entry)) … … 1454 1452 } 1455 1453 1456 ASSERT(count == m_keyCount);1457 ASSERT(deletedCount == m_deletedCount);1458 ASSERT( m_tableSize>= KeyTraits::minimumTableSize);1459 ASSERT( m_tableSizeMask);1460 ASSERT( m_tableSize == m_tableSizeMask+ 1);1454 ASSERT(count == keyCount()); 1455 ASSERT(deletedCount == this->deletedCount()); 1456 ASSERT(this->tableSize() >= KeyTraits::minimumTableSize); 1457 ASSERT(tableSizeMask()); 1458 ASSERT(this->tableSize() == tableSizeMask() + 1); 1461 1459 } 1462 1460 -
trunk/Tools/ChangeLog
r255605 r255611 1 2020-02-03 Alex Christensen <achristensen@webkit.org> 2 3 Reduce size of HashMap and HashSet 4 https://bugs.webkit.org/show_bug.cgi?id=207138 5 6 Reviewed by Yusuke Suzuki. 7 8 * TestWebKitAPI/Tests/WTF/HashMap.cpp: 9 (TestWebKitAPI::TEST): 10 * TestWebKitAPI/Tests/WTF/HashSet.cpp: 11 (TestWebKitAPI::TEST): 12 1 13 2020-02-03 Alexey Shvayka <shvaikalesh@gmail.com> 2 14 -
trunk/Tools/TestWebKitAPI/Tests/WTF/HashMap.cpp
r254881 r255611 89 89 90 90 DoubleHashMap map; 91 #if !CHECK_HASHTABLE_ITERATORS &&!DUMP_HASHTABLE_STATS_PER_TABLE 92 static_assert(sizeof(map) == sizeof(void*)); 93 #endif 91 94 92 95 map.add(clobberKey, 1); -
trunk/Tools/TestWebKitAPI/Tests/WTF/HashSet.cpp
r254881 r255611 171 171 172 172 HashSet<std::unique_ptr<ConstructorDestructorCounter>> set; 173 #if !CHECK_HASHTABLE_ITERATORS &&!DUMP_HASHTABLE_STATS_PER_TABLE 174 static_assert(sizeof(set) == sizeof(void*)); 175 #endif 173 176 174 177 auto uniquePtr = makeUnique<ConstructorDestructorCounter>();
Note: See TracChangeset
for help on using the changeset viewer.