Changeset 179804 in webkit


Ignore:
Timestamp:
Feb 8, 2015 2:06:06 PM (9 years ago)
Author:
Chris Dumez
Message:

[WK2] Add logging to validate the network cache efficacy (Part 1)
​https://bugs.webkit.org/show_bug.cgi?id=141269
<rdar://problem/19632080>

Reviewed by Antti Koivisto.

Source/WebCore:

Export an extra symbol.

  • WebCore.exp.in:

Source/WebKit2:

Add console logging to validate the network cache efficacy. This will
tell us if how the network cache satisties requests, in particular:

  • Request cannot be handled by the cache
  • Entry was not in the cache but is no longer there (pruned)
  • Entry is in the cache but is not usable
  • Entry is in the cache and is used.

This patch introduces a SQLite-based network cache statistics storage
that is used to store requests we have seen before, and query if we
have seen a request before. The storage is lightweight as it only
stores hashes in the database, in a background thread.

The statistics cache is initially bootstapped from the network disk
cache so that we have data initially and get as accurate statistics
as possible from the start.

To maintain an acceptable level of performance, we have a hard limit
on the number of unique requests that are retained set to 100000.

Diagnostic logging for this will be added in a follow-up patch.

Location:
trunk/Source
Files:
3 added
12 edited

Legend:

Unmodified
Added
Removed

  • trunk/Source/WebCore/ChangeLog

    r179796 r179804  
     12015-02-08  Chris Dumez  <cdumez@apple.com>
     2
     3        [WK2] Add logging to validate the network cache efficacy (Part 1)
     4        https://bugs.webkit.org/show_bug.cgi?id=141269
     5        <rdar://problem/19632080>
     6
     7        Reviewed by Antti Koivisto.
     8
     9        Export an extra symbol.
     10
     11        * WebCore.exp.in:
     12
    1132015-02-07  Chris Fleizach  <cfleizach@apple.com>
    214

  • trunk/Source/WebCore/WebCore.exp.in

    r179792 r179804  
    22632263
    22642264#if !ASSERT_DISABLED
     2265__ZN7WebCore21SQLiteDatabaseTracker24hasTransactionInProgressEv
    22652266__ZN7WebCore27NoExceptionAssertionCheckerC1EPKci
    22662267__ZN7WebCore27NoExceptionAssertionCheckerD1Ev

  • trunk/Source/WebKit2/ChangeLog

    r179791 r179804  
     12015-02-08  Chris Dumez  <cdumez@apple.com>
     2
     3        [WK2] Add logging to validate the network cache efficacy (Part 1)
     4        https://bugs.webkit.org/show_bug.cgi?id=141269
     5        <rdar://problem/19632080>
     6
     7        Reviewed by Antti Koivisto.
     8
     9        Add console logging to validate the network cache efficacy. This will
     10        tell us if how the network cache satisties requests, in particular:
     11        - Request cannot be handled by the cache
     12        - Entry was not in the cache but is no longer there (pruned)
     13        - Entry is in the cache but is not usable
     14        - Entry is in the cache and is used.
     15
     16        This patch introduces a SQLite-based network cache statistics storage
     17        that is used to store requests we have seen before, and query if we
     18        have seen a request before. The storage is lightweight as it only
     19        stores hashes in the database, in a background thread.
     20
     21        The statistics cache is initially bootstapped from the network disk
     22        cache so that we have data initially and get as accurate statistics
     23        as possible from the start.
     24
     25        To maintain an acceptable level of performance, we have a hard limit
     26        on the number of unique requests that are retained set to 100000.
     27
     28        Diagnostic logging for this will be added in a follow-up patch.
     29
    1302015-02-07  Chris Dumez  <cdumez@apple.com>
    231

  • trunk/Source/WebKit2/NetworkProcess/cache/NetworkCache.cpp

    r179780 r179804  
    3131#include "Logging.h"
    3232#include "NetworkCacheCoders.h"
     33#include "NetworkCacheStatistics.h"
    3334#include "NetworkCacheStorage.h"
    3435#include "NetworkResourceLoader.h"
    … …  
    5051}
    5152
    52 bool NetworkCache::initialize(const String& cachePath)
     53bool NetworkCache::initialize(const String& cachePath, bool enableEfficacyLogging)
    5354{
    5455    m_storage = NetworkCacheStorage::open(cachePath);
     56
     57    if (enableEfficacyLogging)
     58        m_statistics = NetworkCacheStatistics::open(cachePath);
    5559
    5660    LOG(NetworkCache, "(NetworkProcess) opened cache storage, success %d", !!m_storage);
    … …  
    221225    LOG(NetworkCache, "(NetworkProcess) retrieving %s priority %u", originalRequest.url().string().ascii().data(), originalRequest.priority());
    222226
     227    NetworkCacheKey storageKey = makeCacheKey(originalRequest);
    223228    if (!canRetrieve(originalRequest)) {
     229        if (m_statistics)
     230            m_statistics->recordNotUsingCacheForRequest(storageKey, originalRequest);
     231
    224232        completionHandler(nullptr);
    225233        return;
    … …  
    227235
    228236    auto startTime = std::chrono::system_clock::now();
    229     NetworkCacheKey storageKey = makeCacheKey(originalRequest);
    230237    unsigned priority = originalRequest.priority();
    231238
    232     m_storage->retrieve(storageKey, priority, [this, originalRequest, completionHandler, startTime](std::unique_ptr<NetworkCacheStorage::Entry> entry) {
     239    m_storage->retrieve(storageKey, priority, [this, originalRequest, completionHandler, startTime, storageKey](std::unique_ptr<NetworkCacheStorage::Entry> entry) {
    233240        if (!entry) {
    234241            LOG(NetworkCache, "(NetworkProcess) not found in storage");
     242
     243            if (m_statistics)
     244                m_statistics->recordRetrievalFailure(storageKey, originalRequest);
     245
    235246            completionHandler(nullptr);
    236247            return false;
    … …  
    238249        auto decodedEntry = decodeStorageEntry(*entry, originalRequest);
    239250        bool success = !!decodedEntry;
     251        if (m_statistics)
     252            m_statistics->recordRetrievedCachedEntry(storageKey, originalRequest, success);
     253
    240254#if !LOG_DISABLED
    241255        auto elapsedMS = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now() - startTime).count();
    … …  
    332346    if (m_storage)
    333347        m_storage->clear();
    334 }
    335 
    336 }
    337 
    338 #endif
     348    if (m_statistics)
     349        m_statistics->clear();
     350}
     351
     352String NetworkCache::storagePath() const
     353{
     354    return m_storage ? m_storage->directoryPath() : String();
     355}
     356
     357}
     358
     359#endif

  • trunk/Source/WebKit2/NetworkProcess/cache/NetworkCache.h

    r179708 r179804  
    4242namespace WebKit {
    4343
     44class NetworkCacheStatistics;
     45
    4446class NetworkCache {
    4547    WTF_MAKE_NONCOPYABLE(NetworkCache);
    … …  
    4850    static NetworkCache& singleton();
    4951
    50     bool initialize(const String& cachePath);
     52    bool initialize(const String& cachePath, bool enableEfficacyLogging);
    5153    void setMaximumSize(size_t);
    5254
    … …  
    7678    void clear();
    7779
     80    String storagePath() const;
     81
    7882private:
    7983    NetworkCache() = default;
    … …  
    8185
    8286    std::unique_ptr<NetworkCacheStorage> m_storage;
     87    std::unique_ptr<NetworkCacheStatistics> m_statistics;
    8388};
    8489

  • trunk/Source/WebKit2/NetworkProcess/cache/NetworkCacheStorage.h

    r179779 r179804  
    156156    static const unsigned version = 2;
    157157
     158    const String& directoryPath() const { return m_directoryPath; }
     159
    158160private:
    159161    NetworkCacheStorage(const String& directoryPath);

  • trunk/Source/WebKit2/NetworkProcess/cache/NetworkCacheStorageCocoa.mm

    r179779 r179804  
    3131#include "Logging.h"
    3232#include "NetworkCacheCoders.h"
    33 #include <WebCore/FileSystem.h>
    34 #include <dirent.h>
     33#include "NetworkCacheFileSystemPosix.h"
    3534#include <dispatch/dispatch.h>
    3635#include <sys/mman.h>
    … …  
    4443static const char networkCacheSubdirectory[] = "WebKitCache";
    4544static const char versionDirectoryPrefix[] = "Version ";
    46 
    47 template <typename Function>
    48 static void traverseDirectory(const String& path, uint8_t type, const Function& function)
    49 {
    50     DIR* dir = opendir(WebCore::fileSystemRepresentation(path).data());
    51     if (!dir)
    52         return;
    53     struct dirent* dp;
    54     while ((dp = readdir(dir))) {
    55         if (dp->d_type != type)
    56             continue;
    57         const char* name = dp->d_name;
    58         if (!strcmp(name, ".") || !strcmp(name, ".."))
    59             continue;
    60         function(String(name));
    61     }
    62     closedir(dir);
    63 }
    64 
    65 static void traverseCacheFiles(const String& cachePath, std::function<void (const String& fileName, const String& partitionPath)> function)
    66 {
    67     traverseDirectory(cachePath, DT_DIR, [&cachePath, &function](const String& subdirName) {
    68         String partitionPath = WebCore::pathByAppendingComponent(cachePath, subdirName);
    69         traverseDirectory(partitionPath, DT_REG, [&function, &partitionPath](const String& fileName) {
    70             if (fileName.length() != NetworkCacheKey::hashStringLength())
    71                 return;
    72             function(fileName, partitionPath);
    73         });
    74     });
    75 }
    7645
    7746NetworkCacheStorage::Data::Data(const uint8_t* data, size_t size)

  • trunk/Source/WebKit2/NetworkProcess/cocoa/NetworkProcessCocoa.mm

    r179409 r179804  
    6565        SandboxExtension::consumePermanently(parameters.diskCacheDirectoryExtensionHandle);
    6666#if ENABLE(NETWORK_CACHE)
    67         if (parameters.shouldEnableNetworkCache && NetworkCache::singleton().initialize(m_diskCacheDirectory)) {
     67        if (parameters.shouldEnableNetworkCache && NetworkCache::singleton().initialize(m_diskCacheDirectory, parameters.shouldEnableNetworkCacheEfficacyLogging)) {
    6868            RetainPtr<NSURLCache> urlCache(adoptNS([[NSURLCache alloc] initWithMemoryCapacity:0 diskCapacity:0 diskPath:nil]));
    6969            [NSURLCache setSharedURLCache:urlCache.get()];

  • trunk/Source/WebKit2/Shared/Network/NetworkProcessCreationParameters.cpp

    r179148 r179804  
    4646#if ENABLE(NETWORK_CACHE)
    4747    encoder << shouldEnableNetworkCache;
     48    encoder << shouldEnableNetworkCacheEfficacyLogging;
    4849#endif
    4950    encoder << cookieStorageDirectory;
    … …  
    8687#if ENABLE(NETWORK_CACHE)
    8788    if (!decoder.decode(result.shouldEnableNetworkCache))
     89        return false;
     90    if (!decoder.decode(result.shouldEnableNetworkCacheEfficacyLogging))
    8891        return false;
    8992#endif

  • trunk/Source/WebKit2/Shared/Network/NetworkProcessCreationParameters.h

    r179148 r179804  
    5959#if ENABLE(NETWORK_CACHE)
    6060    bool shouldEnableNetworkCache;
     61    bool shouldEnableNetworkCacheEfficacyLogging;
    6162#endif
    6263

  • trunk/Source/WebKit2/UIProcess/Cocoa/WebProcessPoolCocoa.mm

    r179409 r179804  
    8282#if ENABLE(NETWORK_CACHE)
    8383static NSString * const WebKitNetworkCacheEnabledDefaultsKey = @"WebKitNetworkCacheEnabled";
     84static NSString * const WebKitNetworkCacheEfficacyLoggingEnabledDefaultsKey = @"WebKitNetworkCacheEfficacyLoggingEnabled";
    8485#endif
    8586
    … …  
    107108#if ENABLE(NETWORK_CACHE)
    108109    [registrationDictionary setObject:[NSNumber numberWithBool:YES] forKey:WebKitNetworkCacheEnabledDefaultsKey];
     110    [registrationDictionary setObject:[NSNumber numberWithBool:YES] forKey:WebKitNetworkCacheEfficacyLoggingEnabledDefaultsKey];
    109111#endif
    110112
    … …  
    253255#if ENABLE(NETWORK_CACHE)
    254256    parameters.shouldEnableNetworkCache = [[NSUserDefaults standardUserDefaults] boolForKey:WebKitNetworkCacheEnabledDefaultsKey];
     257    parameters.shouldEnableNetworkCacheEfficacyLogging = [[NSUserDefaults standardUserDefaults] boolForKey:WebKitNetworkCacheEfficacyLoggingEnabledDefaultsKey];
    255258#endif
    256259}

  • trunk/Source/WebKit2/WebKit2.xcodeproj/project.pbxproj

    r179786 r179804  
    11481148                7CF47FFF17276AE3008ACB91 /* WKBundlePageBannerMac.h in Headers */ = {isa = PBXBuildFile; fileRef = 7CF47FFD17276AE3008ACB91 /* WKBundlePageBannerMac.h */; settings = {ATTRIBUTES = (Private, ); }; };
    11491149                7EC4F0FB18E4ACBB008056AF /* NetworkProcessCocoa.mm in Sources */ = {isa = PBXBuildFile; fileRef = 7EC4F0F918E4A945008056AF /* NetworkProcessCocoa.mm */; };
     1150                834B250F1A831A8D00CFB150 /* NetworkCacheFileSystemPosix.h in Headers */ = {isa = PBXBuildFile; fileRef = 834B250E1A831A8D00CFB150 /* NetworkCacheFileSystemPosix.h */; };
     1151                834B25121A842C8700CFB150 /* NetworkCacheStatistics.h in Headers */ = {isa = PBXBuildFile; fileRef = 834B25101A842C8700CFB150 /* NetworkCacheStatistics.h */; };
     1152                834B25131A842C8700CFB150 /* NetworkCacheStatisticsCocoa.mm in Sources */ = {isa = PBXBuildFile; fileRef = 834B25111A842C8700CFB150 /* NetworkCacheStatisticsCocoa.mm */; };
    11501153                8372DB251A674C8F00C697C5 /* WKPageDiagnosticLoggingClient.h in Headers */ = {isa = PBXBuildFile; fileRef = 8372DB241A674C8F00C697C5 /* WKPageDiagnosticLoggingClient.h */; settings = {ATTRIBUTES = (Private, ); }; };
    11511154                8372DB281A67562800C697C5 /* WebPageDiagnosticLoggingClient.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 8372DB261A67562800C697C5 /* WebPageDiagnosticLoggingClient.cpp */; };
    … …  
    33223325                7CF47FFD17276AE3008ACB91 /* WKBundlePageBannerMac.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = WKBundlePageBannerMac.h; sourceTree = "<group>"; };
    33233326                7EC4F0F918E4A945008056AF /* NetworkProcessCocoa.mm */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.objcpp; name = NetworkProcessCocoa.mm; path = NetworkProcess/cocoa/NetworkProcessCocoa.mm; sourceTree = "<group>"; };
     3327                834B250E1A831A8D00CFB150 /* NetworkCacheFileSystemPosix.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = NetworkCacheFileSystemPosix.h; sourceTree = "<group>"; };
     3328                834B25101A842C8700CFB150 /* NetworkCacheStatistics.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = NetworkCacheStatistics.h; sourceTree = "<group>"; };
     3329                834B25111A842C8700CFB150 /* NetworkCacheStatisticsCocoa.mm */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.objcpp; path = NetworkCacheStatisticsCocoa.mm; sourceTree = "<group>"; };
    33243330                8372DB241A674C8F00C697C5 /* WKPageDiagnosticLoggingClient.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = WKPageDiagnosticLoggingClient.h; sourceTree = "<group>"; };
    33253331                8372DB261A67562800C697C5 /* WebPageDiagnosticLoggingClient.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = WebPageDiagnosticLoggingClient.cpp; sourceTree = "<group>"; };
    … …  
    73837389                                E489D2881A0A2DB80078C06A /* NetworkCacheEncoder.cpp */,
    73847390                                E489D2891A0A2DB80078C06A /* NetworkCacheEncoder.h */,
     7391                                834B250E1A831A8D00CFB150 /* NetworkCacheFileSystemPosix.h */,
    73857392                                E4436EC01A0CFDB200EAD204 /* NetworkCacheKey.cpp */,
    73867393                                E4436EC11A0CFDB200EAD204 /* NetworkCacheKey.h */,
     7394                                834B25101A842C8700CFB150 /* NetworkCacheStatistics.h */,
     7395                                834B25111A842C8700CFB150 /* NetworkCacheStatisticsCocoa.mm */,
    73877396                                E4436EC21A0CFDB200EAD204 /* NetworkCacheStorage.h */,
    73887397                                E4436EC31A0CFDB200EAD204 /* NetworkCacheStorageCocoa.mm */,
    … …  
    75017510                                1AFDD3151891B54000153970 /* APIPolicyClient.h in Headers */,
    75027511                                7CE4D2201A4914CA00C7F152 /* APIProcessPoolConfiguration.h in Headers */,
     7512                                834B25121A842C8700CFB150 /* NetworkCacheStatistics.h in Headers */,
    75037513                                F634445612A885C8000612D8 /* APISecurityOrigin.h in Headers */,
    75047514                                75A8D2E1187DEC1A00C39C9E /* APISession.h in Headers */,
    … …  
    76877697                                1A2162B111F38971008AD0F5 /* NPRuntimeUtilities.h in Headers */,
    76887698                                1A2D84A3127F6AD1001EB962 /* NPVariantData.h in Headers */,
     7699                                834B250F1A831A8D00CFB150 /* NetworkCacheFileSystemPosix.h in Headers */,
    76897700                                BC8ACA1316670D89004C1941 /* ObjCObjectGraph.h in Headers */,
    76907701                                BCCF672D12C7EDF7008F9C35 /* OriginAndDatabases.h in Headers */,
    … …  
    98869897                                0F3C725C196F605200AEDD0C /* WKInspectorHighlightView.mm in Sources */,
    98879898                                A54293A5195A43DD002782C7 /* WKInspectorNodeSearchGestureRecognizer.mm in Sources */,
     9899                                834B25131A842C8700CFB150 /* NetworkCacheStatisticsCocoa.mm in Sources */,
    98889900                                51A9E10A1315CD18009E7031 /* WKKeyValueStorageManager.cpp in Sources */,
    98899901                                33D3A3B51339600B00709BE4 /* WKMediaCacheManager.cpp in Sources */,
Note: See TracChangeset for help on using the changeset viewer.