Changeset 182954 in webkit
- Timestamp:
- Apr 17, 2015 10:07:18 AM (9 years ago)
- Location:
- trunk/Source/WebKit2
- Files:
-
- 4 edited
Legend:
- Unmodified
- Added
- Removed
-
trunk/Source/WebKit2/ChangeLog
r182952 r182954 1 2015-04-17 Antti Koivisto <antti@apple.com> 2 3 Network Cache: Read resource record and body in parallel 4 https://bugs.webkit.org/show_bug.cgi?id=143879 5 6 Reviewed by Chris Dumez. 7 8 We currently first fetch the record file and then fetch the body blob if needed. 9 We can do both operations in parallel to reduce latency. 10 11 * NetworkProcess/cache/NetworkCacheFileSystemPosix.h: 12 (WebKit::NetworkCache::traverseCacheFiles): 13 14 Do all validation in the client. 15 16 * NetworkProcess/cache/NetworkCacheStorage.cpp: 17 (WebKit::NetworkCache::Storage::synchronize): 18 19 Maintain a bloom filter that contains the body blobs to avoid unnecessary IO attempts. 20 Delete any unknown file in cache directory. 21 22 (WebKit::NetworkCache::Storage::addToRecordFilter): 23 24 More informative name for record filter. 25 26 (WebKit::NetworkCache::Storage::mayContain): 27 (WebKit::NetworkCache::Storage::readRecord): 28 (WebKit::NetworkCache::Storage::storeBodyAsBlob): 29 (WebKit::NetworkCache::Storage::dispatchReadOperation): 30 31 Start record read IO and body blob read IO in parallel. 32 33 (WebKit::NetworkCache::Storage::finishReadOperation): 34 35 The read is finished when we have both the record and the blob. 36 37 (WebKit::NetworkCache::Storage::dispatchWriteOperation): 38 (WebKit::NetworkCache::Storage::retrieve): 39 (WebKit::NetworkCache::Storage::store): 40 (WebKit::NetworkCache::Storage::traverse): 41 (WebKit::NetworkCache::Storage::clear): 42 (WebKit::NetworkCache::Storage::shrink): 43 (WebKit::NetworkCache::Storage::addToContentsFilter): Deleted. 44 (WebKit::NetworkCache::Storage::decodeRecord): Deleted. 45 * NetworkProcess/cache/NetworkCacheStorage.h: 46 (WebKit::NetworkCache::Storage::ReadOperation::ReadOperation): 47 48 ReadOperation is now mutable and gathers the read result. 49 1 50 2015-04-16 Anders Carlsson <andersca@apple.com> 2 51 -
trunk/Source/WebKit2/NetworkProcess/cache/NetworkCacheFileSystemPosix.h
r182946 r182954 29 29 #if ENABLE(NETWORK_CACHE) 30 30 31 #include "NetworkCacheKey.h"32 31 #include <WebCore/FileSystem.h> 33 32 #include <dirent.h> … … 63 62 String partitionPath = WebCore::pathByAppendingComponent(cachePath, subdirName); 64 63 traverseDirectory(partitionPath, DT_REG, [&function, &partitionPath](const String& fileName) { 65 if (fileName.length() != Key::hashStringLength())66 return;67 64 function(fileName, partitionPath); 68 65 }); -
trunk/Source/WebKit2/NetworkProcess/cache/NetworkCacheStorage.cpp
r182856 r182954 106 106 size_t Storage::approximateSize() const 107 107 { 108 return m_approximate Size + m_blobStorage.approximateSize();108 return m_approximateRecordsSize + m_blobStorage.approximateSize(); 109 109 } 110 110 … … 120 120 121 121 backgroundIOQueue().dispatch([this] { 122 auto filter = std::make_unique<ContentsFilter>(); 123 size_t size = 0; 122 auto recordFilter = std::make_unique<ContentsFilter>(); 123 auto bodyFilter = std::make_unique<ContentsFilter>(); 124 size_t recordsSize = 0; 124 125 unsigned count = 0; 125 traverseCacheFiles(recordsPath(), [&filter, &size, &count](const String& fileName, const String& partitionPath) { 126 traverseCacheFiles(recordsPath(), [&recordFilter, &bodyFilter, &recordsSize, &count](const String& fileName, const String& partitionPath) { 127 auto filePath = WebCore::pathByAppendingComponent(partitionPath, fileName); 128 129 bool isBody = fileName.endsWith(bodyPostfix); 130 String hashString = isBody ? fileName.substring(0, Key::hashStringLength()) : fileName; 126 131 Key::HashType hash; 127 if (!Key::stringToHash(fileName, hash)) 132 if (!Key::stringToHash(hashString, hash)) { 133 WebCore::deleteFile(filePath); 128 134 return; 129 auto filePath = WebCore::pathByAppendingComponent(partitionPath, fileName);135 } 130 136 long long fileSize = 0; 131 137 WebCore::getFileSize(filePath, fileSize); 132 if (!fileSize) 138 if (!fileSize) { 139 WebCore::deleteFile(filePath); 133 140 return; 134 filter->add(hash); 135 size += fileSize; 141 } 142 if (isBody) { 143 bodyFilter->add(hash); 144 return; 145 } 146 recordFilter->add(hash); 147 recordsSize += fileSize; 136 148 ++count; 137 149 }); 138 150 139 auto* filterPtr = filter.release(); 140 RunLoop::main().dispatch([this, filterPtr, size] { 141 auto filter = std::unique_ptr<ContentsFilter>(filterPtr); 142 143 for (auto hash : m_contentsFilterHashesAddedDuringSynchronization) 144 filter->add(hash); 145 m_contentsFilterHashesAddedDuringSynchronization.clear(); 146 147 m_contentsFilter = WTF::move(filter); 148 m_approximateSize = size; 151 auto* recordFilterPtr = recordFilter.release(); 152 auto* bodyFilterPtr = bodyFilter.release(); 153 RunLoop::main().dispatch([this, recordFilterPtr, bodyFilterPtr, recordsSize] { 154 auto recordFilter = std::unique_ptr<ContentsFilter>(recordFilterPtr); 155 auto bodyFilter = std::unique_ptr<ContentsFilter>(bodyFilterPtr); 156 157 for (auto hash : m_recordFilterHashesAddedDuringSynchronization) 158 recordFilter->add(hash); 159 m_recordFilterHashesAddedDuringSynchronization.clear(); 160 161 for (auto hash : m_bodyFilterHashesAddedDuringSynchronization) 162 bodyFilter->add(hash); 163 m_bodyFilterHashesAddedDuringSynchronization.clear(); 164 165 m_recordFilter = WTF::move(recordFilter); 166 m_bodyFilter = WTF::move(bodyFilter); 167 m_approximateRecordsSize = recordsSize; 149 168 m_synchronizationInProgress = false; 150 169 }); … … 152 171 m_blobStorage.synchronize(); 153 172 154 LOG(NetworkCacheStorage, "(NetworkProcess) cache synchronization completed size=%zu count=%d", size, count);155 }); 156 } 157 158 void Storage::addTo ContentsFilter(const Key& key)159 { 160 ASSERT(RunLoop::isMain()); 161 162 if (m_ contentsFilter)163 m_ contentsFilter->add(key.hash());173 LOG(NetworkCacheStorage, "(NetworkProcess) cache synchronization completed size=%zu count=%d", recordsSize, count); 174 }); 175 } 176 177 void Storage::addToRecordFilter(const Key& key) 178 { 179 ASSERT(RunLoop::isMain()); 180 181 if (m_recordFilter) 182 m_recordFilter->add(key.hash()); 164 183 165 184 // If we get new entries during filter synchronization take care to add them to the new filter as well. 166 185 if (m_synchronizationInProgress) 167 m_ contentsFilterHashesAddedDuringSynchronization.append(key.hash());186 m_recordFilterHashesAddedDuringSynchronization.append(key.hash()); 168 187 } 169 188 … … 171 190 { 172 191 ASSERT(RunLoop::isMain()); 173 return !m_ contentsFilter || m_contentsFilter->mayContain(key.hash());192 return !m_recordFilter || m_recordFilter->mayContain(key.hash()); 174 193 } 175 194 … … 279 298 } 280 299 281 std::unique_ptr<Storage::Record> Storage::decodeRecord(const Data& recordData, const Key& key)300 void Storage::readRecord(ReadOperation& readOperation, const Data& recordData) 282 301 { 283 302 ASSERT(!RunLoop::isMain()); … … 286 305 Data headerData; 287 306 if (!decodeRecordHeader(recordData, metaData, headerData)) 288 return nullptr;289 290 if (metaData.key != key)291 return nullptr;307 return; 308 309 if (metaData.key != readOperation.key) 310 return; 292 311 293 312 // Sanity check against time stamps in future. 294 313 auto timeStamp = std::chrono::system_clock::time_point(metaData.epochRelativeTimeStamp); 295 314 if (timeStamp > std::chrono::system_clock::now()) 296 return nullptr;315 return; 297 316 298 317 Data bodyData; … … 300 319 size_t bodyOffset = metaData.headerOffset + headerData.size(); 301 320 if (bodyOffset + metaData.bodySize != recordData.size()) 302 return nullptr;321 return; 303 322 bodyData = recordData.subrange(bodyOffset, metaData.bodySize); 304 323 if (metaData.bodyHash != computeSHA1(bodyData)) 305 return nullptr; 306 } else { 307 auto bodyPath = bodyPathForKey(key, recordsPath()); 308 auto bodyBlob = m_blobStorage.get(bodyPath); 309 if (metaData.bodySize != bodyBlob.data.size()) 310 return nullptr; 311 if (metaData.bodyHash != bodyBlob.hash) 312 return nullptr; 313 bodyData = bodyBlob.data; 314 } 315 316 return std::make_unique<Storage::Record>(Storage::Record { 324 return; 325 } 326 327 readOperation.expectedBodyHash = metaData.bodyHash; 328 readOperation.resultRecord = std::make_unique<Storage::Record>(Storage::Record { 317 329 metaData.key, 318 330 timeStamp, … … 349 361 return { }; 350 362 351 // Tell the client we now have a disk-backed map for this data. 352 if (mappedBodyHandler) { 353 RunLoop::main().dispatch([blob, mappedBodyHandler] { 363 auto hash = record.key.hash(); 364 RunLoop::main().dispatch([this, blob, hash, mappedBodyHandler] { 365 if (m_bodyFilter) 366 m_bodyFilter->add(hash); 367 if (m_synchronizationInProgress) 368 m_bodyFilterHashesAddedDuringSynchronization.append(hash); 369 370 if (mappedBodyHandler) 354 371 mappedBodyHandler(blob.data); 355 }); 356 } 372 373 }); 357 374 return blob; 358 375 } … … 402 419 } 403 420 404 void Storage::dispatchReadOperation(const ReadOperation& read) 405 { 406 ASSERT(RunLoop::isMain()); 407 ASSERT(m_activeReadOperations.contains(&read)); 408 409 auto recordsPath = this->recordsPath(); 410 auto recordPath = recordPathForKey(read.key, recordsPath); 421 void Storage::dispatchReadOperation(ReadOperation& readOperation) 422 { 423 ASSERT(RunLoop::isMain()); 424 ASSERT(m_activeReadOperations.contains(&readOperation)); 425 426 auto recordPath = recordPathForKey(readOperation.key, recordsPath()); 411 427 412 428 RefPtr<IOChannel> channel = IOChannel::open(recordPath, IOChannel::Type::Read); 413 channel->read(0, std::numeric_limits<size_t>::max(), &ioQueue(), [this, &read](const Data& fileData, int error) { 414 auto record = error ? nullptr : decodeRecord(fileData, read.key); 415 416 auto* recordPtr = record.release(); 417 RunLoop::main().dispatch([this, &read, recordPtr] { 418 auto record = std::unique_ptr<Record>(recordPtr); 419 finishReadOperation(read, WTF::move(record)); 420 }); 421 }); 422 } 423 424 void Storage::finishReadOperation(const ReadOperation& read, std::unique_ptr<Record> record) 425 { 426 ASSERT(RunLoop::isMain()); 427 428 bool success = read.completionHandler(WTF::move(record)); 429 if (success) 430 updateFileModificationTime(recordPathForKey(read.key, recordsPath())); 431 else 432 remove(read.key); 433 ASSERT(m_activeReadOperations.contains(&read)); 434 m_activeReadOperations.remove(&read); 435 dispatchPendingReadOperations(); 436 437 LOG(NetworkCacheStorage, "(NetworkProcess) read complete success=%d", success); 429 channel->read(0, std::numeric_limits<size_t>::max(), &ioQueue(), [this, &readOperation](const Data& fileData, int error) { 430 if (!error) 431 readRecord(readOperation, fileData); 432 finishReadOperation(readOperation); 433 }); 434 435 bool shouldGetBodyBlob = !m_bodyFilter || m_bodyFilter->mayContain(readOperation.key.hash()); 436 if (!shouldGetBodyBlob) { 437 finishReadOperation(readOperation); 438 return; 439 } 440 441 // Read the body blob in parallel with the record read. 442 ioQueue().dispatch([this, &readOperation] { 443 auto bodyPath = bodyPathForKey(readOperation.key, this->recordsPath()); 444 readOperation.resultBodyBlob = m_blobStorage.get(bodyPath); 445 finishReadOperation(readOperation); 446 }); 447 } 448 449 void Storage::finishReadOperation(ReadOperation& readOperation) 450 { 451 // Record and body blob reads must finish. 452 bool isComplete = ++readOperation.finishedCount == 2; 453 if (!isComplete) 454 return; 455 456 RunLoop::main().dispatch([this, &readOperation] { 457 if (readOperation.resultRecord && readOperation.resultRecord->body.isNull()) { 458 if (readOperation.resultBodyBlob.hash == readOperation.expectedBodyHash) 459 readOperation.resultRecord->body = readOperation.resultBodyBlob.data; 460 else 461 readOperation.resultRecord = nullptr; 462 } 463 464 bool success = readOperation.completionHandler(WTF::move(readOperation.resultRecord)); 465 if (success) 466 updateFileModificationTime(recordPathForKey(readOperation.key, recordsPath())); 467 else 468 remove(readOperation.key); 469 ASSERT(m_activeReadOperations.contains(&readOperation)); 470 m_activeReadOperations.remove(&readOperation); 471 dispatchPendingReadOperations(); 472 473 LOG(NetworkCacheStorage, "(NetworkProcess) read complete success=%d", success); 474 }); 438 475 } 439 476 … … 499 536 } 500 537 501 void Storage::dispatchWriteOperation(const WriteOperation& write )502 { 503 ASSERT(RunLoop::isMain()); 504 ASSERT(m_activeWriteOperations.contains(&write ));538 void Storage::dispatchWriteOperation(const WriteOperation& writeOperation) 539 { 540 ASSERT(RunLoop::isMain()); 541 ASSERT(m_activeWriteOperations.contains(&writeOperation)); 505 542 506 543 // This was added already when starting the store but filter might have been wiped. 507 addTo ContentsFilter(write.record.key);508 509 backgroundIOQueue().dispatch([this, &write ] {544 addToRecordFilter(writeOperation.record.key); 545 546 backgroundIOQueue().dispatch([this, &writeOperation] { 510 547 auto recordsPath = this->recordsPath(); 511 auto partitionPath = partitionPathForKey(write .record.key, recordsPath);512 auto recordPath = recordPathForKey(write .record.key, recordsPath);548 auto partitionPath = partitionPathForKey(writeOperation.record.key, recordsPath); 549 auto recordPath = recordPathForKey(writeOperation.record.key, recordsPath); 513 550 514 551 WebCore::makeAllDirectories(partitionPath); 515 552 516 bool shouldStoreAsBlob = shouldStoreBodyAsBlob(write .record.body);517 auto bodyBlob = shouldStoreAsBlob ? storeBodyAsBlob(write .record, write.mappedBodyHandler) : Nullopt;518 519 auto recordData = encodeRecord(write .record, bodyBlob);553 bool shouldStoreAsBlob = shouldStoreBodyAsBlob(writeOperation.record.body); 554 auto bodyBlob = shouldStoreAsBlob ? storeBodyAsBlob(writeOperation.record, writeOperation.mappedBodyHandler) : Nullopt; 555 556 auto recordData = encodeRecord(writeOperation.record, bodyBlob); 520 557 521 558 auto channel = IOChannel::open(recordPath, IOChannel::Type::Create); 522 559 size_t recordSize = recordData.size(); 523 channel->write(0, recordData, nullptr, [this, &write , recordSize](int error) {560 channel->write(0, recordData, nullptr, [this, &writeOperation, recordSize](int error) { 524 561 // On error the entry still stays in the contents filter until next synchronization. 525 m_approximate Size += recordSize;526 finishWriteOperation(write );562 m_approximateRecordsSize += recordSize; 563 finishWriteOperation(writeOperation); 527 564 528 565 LOG(NetworkCacheStorage, "(NetworkProcess) write complete error=%d", error); … … 531 568 } 532 569 533 void Storage::finishWriteOperation(const WriteOperation& write )534 { 535 ASSERT(m_activeWriteOperations.contains(&write ));536 m_activeWriteOperations.remove(&write );570 void Storage::finishWriteOperation(const WriteOperation& writeOperation) 571 { 572 ASSERT(m_activeWriteOperations.contains(&writeOperation)); 573 m_activeWriteOperations.remove(&writeOperation); 537 574 dispatchPendingWriteOperations(); 538 575 … … 561 598 return; 562 599 563 m_pendingReadOperationsByPriority[priority].append(new ReadOperation { key, WTF::move(completionHandler) } );600 m_pendingReadOperationsByPriority[priority].append(new ReadOperation { key, WTF::move(completionHandler) } ); 564 601 dispatchPendingReadOperations(); 565 602 } … … 576 613 577 614 // Add key to the filter already here as we do lookups from the pending operations too. 578 addTo ContentsFilter(record.key);615 addToRecordFilter(record.key); 579 616 580 617 dispatchPendingWriteOperations(); … … 585 622 ioQueue().dispatch([this, flags, traverseHandler] { 586 623 traverseCacheFiles(recordsPath(), [this, flags, &traverseHandler](const String& fileName, const String& partitionPath) { 624 if (fileName.length() != Key::hashStringLength()) 625 return; 587 626 auto recordPath = WebCore::pathByAppendingComponent(partitionPath, fileName); 588 627 … … 635 674 LOG(NetworkCacheStorage, "(NetworkProcess) clearing cache"); 636 675 637 if (m_contentsFilter) 638 m_contentsFilter->clear(); 639 m_approximateSize = 0; 676 if (m_recordFilter) 677 m_recordFilter->clear(); 678 if (m_bodyFilter) 679 m_bodyFilter->clear(); 680 m_approximateRecordsSize = 0; 640 681 641 682 ioQueue().dispatch([this] { … … 709 750 auto recordsPath = this->recordsPath(); 710 751 traverseCacheFiles(recordsPath, [this](const String& fileName, const String& partitionPath) { 752 if (fileName.length() != Key::hashStringLength()) 753 return; 711 754 auto recordPath = WebCore::pathByAppendingComponent(partitionPath, fileName); 712 755 auto bodyPath = bodyPathForRecordPath(recordPath); -
trunk/Source/WebKit2/NetworkProcess/cache/NetworkCacheStorage.h
r182856 r182954 97 97 98 98 struct ReadOperation { 99 Key key; 100 RetrieveCompletionHandler completionHandler; 99 ReadOperation(const Key& key, const RetrieveCompletionHandler& completionHandler) 100 : key(key) 101 , completionHandler(completionHandler) 102 { } 103 104 const Key key; 105 const RetrieveCompletionHandler completionHandler; 106 107 std::unique_ptr<Record> resultRecord; 108 SHA1::Digest expectedBodyHash; 109 BlobStorage::Blob resultBodyBlob; 110 std::atomic<unsigned> finishedCount { 0 }; 101 111 }; 102 void dispatchReadOperation( constReadOperation&);112 void dispatchReadOperation(ReadOperation&); 103 113 void dispatchPendingReadOperations(); 104 void finishReadOperation( const ReadOperation&, std::unique_ptr<Record>);114 void finishReadOperation(ReadOperation&); 105 115 106 116 struct WriteOperation { … … 114 124 Optional<BlobStorage::Blob> storeBodyAsBlob(const Record&, const MappedBodyHandler&); 115 125 Data encodeRecord(const Record&, Optional<BlobStorage::Blob>); 116 std::unique_ptr<Record> decodeRecord(const Data&, const Key&);126 void readRecord(ReadOperation&, const Data&); 117 127 118 128 void updateFileModificationTime(const String& path); … … 124 134 bool mayContain(const Key&) const; 125 135 126 void addTo ContentsFilter(const Key&);136 void addToRecordFilter(const Key&); 127 137 128 138 const String m_basePath; … … 130 140 131 141 size_t m_capacity { std::numeric_limits<size_t>::max() }; 132 size_t m_approximate Size { 0 };142 size_t m_approximateRecordsSize { 0 }; 133 143 134 144 // 2^18 bit filter can support up to 26000 entries with false positive rate < 1%. 135 145 using ContentsFilter = BloomFilter<18>; 136 std::unique_ptr<ContentsFilter> m_contentsFilter; 146 std::unique_ptr<ContentsFilter> m_recordFilter; 147 std::unique_ptr<ContentsFilter> m_bodyFilter; 137 148 138 149 bool m_synchronizationInProgress { false }; 139 150 bool m_shrinkInProgress { false }; 140 151 141 Vector<Key::HashType> m_contentsFilterHashesAddedDuringSynchronization; 152 Vector<Key::HashType> m_recordFilterHashesAddedDuringSynchronization; 153 Vector<Key::HashType> m_bodyFilterHashesAddedDuringSynchronization; 142 154 143 155 static const int maximumRetrievePriority = 4; 144 Deque<std::unique_ptr< constReadOperation>> m_pendingReadOperationsByPriority[maximumRetrievePriority + 1];145 HashSet<std::unique_ptr< constReadOperation>> m_activeReadOperations;156 Deque<std::unique_ptr<ReadOperation>> m_pendingReadOperationsByPriority[maximumRetrievePriority + 1]; 157 HashSet<std::unique_ptr<ReadOperation>> m_activeReadOperations; 146 158 147 159 Deque<std::unique_ptr<const WriteOperation>> m_pendingWriteOperations;
Note: See TracChangeset
for help on using the changeset viewer.