⚠ Archived content — this site is no longer maintained.   Current WebKit documentation is at docs.webkit.org.

Changeset 270607 in webkit


Ignore:
Timestamp:
Dec 9, 2020, 3:13:28 PM (6 years ago)
Author:
Dewei Zhu
Message:

Add max age for a root to be reused.
​https://bugs.webkit.org/show_bug.cgi?id=219628

Reviewed by Ryosuke Niwa.

In order to prevent reusing a stale root, we should set a limit on the age of a root to be reused.

  • public/include/manifest-generator.php: Added 'maxRootReuseAgeInDays' to manifest.
  • public/v3/models/build-request.js: Added root age check.

(BuildRequest.prototype.async findBuildRequestWithSameRoots):

  • public/v3/models/commit-set.js: Extended 'areAllRootsAvailable' with root age check.

(CommitSet.prototype.areAllRootsAvailable):

  • public/v3/models/manifest.js:

(Manifest.fetch): Made it async.
(Manifest.async fetchRawResponse): Extract fetching raw manifest out so that 'maxRootReuseAgeInDays'
can be read without resetting other data models. Also added code to only fetch from API if requesting
/data/manifest.json returns 404.
(Manifest._didFetchManifest):

  • server-tests/api-manifest-tests.js: Updated unit tests.
  • unit-tests/build-request-tests.js: Updated unit tests and add new tests.
  • unit-tests/manifest-test.js: Added unit tests.
Location:
trunk/Websites/perf.webkit.org
Files:
1 added
7 edited

Legend:

Unmodified
Added
Removed

  • trunk/Websites/perf.webkit.org/ChangeLog

    r270140 r270607  
     12020-12-09  Dewei Zhu  <dewei_zhu@apple.com>
     2
     3        Add max age for a root to be reused.
     4        https://bugs.webkit.org/show_bug.cgi?id=219628
     5
     6        Reviewed by Ryosuke Niwa.
     7
     8        In order to prevent reusing a stale root, we should set a limit on the age of a root to be reused.
     9        * public/include/manifest-generator.php: Added 'maxRootReuseAgeInDays' to manifest.
     10        * public/v3/models/build-request.js: Added root age check.
     11        (BuildRequest.prototype.async findBuildRequestWithSameRoots):
     12        * public/v3/models/commit-set.js: Extended 'areAllRootsAvailable' with root age check.
     13        (CommitSet.prototype.areAllRootsAvailable):
     14        * public/v3/models/manifest.js:
     15        (Manifest.fetch): Made it async.
     16        (Manifest.async fetchRawResponse): Extract fetching raw manifest out so that 'maxRootReuseAgeInDays'
     17        can be read without resetting other data models. Also added code to only fetch from API if requesting
     18        /data/manifest.json returns 404.
     19        (Manifest._didFetchManifest):
     20        * server-tests/api-manifest-tests.js: Updated unit tests.
     21        * unit-tests/build-request-tests.js: Updated unit tests and add new tests.
     22        * unit-tests/manifest-test.js: Added unit tests.
     23
    1242020-11-20  Dewei Zhu  <dewei_zhu@apple.com>
    225

  • trunk/Websites/perf.webkit.org/public/include/manifest-generator.php

    r269083 r270607  
    5151            'fileUploadSizeLimit' => config('uploadFileLimitInMB', 0) * 1024 * 1024,
    5252            'testAgeToleranceInHours' => config('testAgeToleranceInHours'),
     53            'maxRootReuseAgeInDays' => config('maxRootReuseAgeInDays'),
    5354        );
    5455

  • trunk/Websites/perf.webkit.org/public/v3/models/build-request.js

    r269871 r270607  
    9696        // Set ignoreCache = true as latest status of test groups is expected.
    9797        const allTestGroupsInTask = await TestGroup.fetchForTask(this.analysisTaskId(), true);
     98        const rawManifest = await Manifest.fetchRawResponse();
     99        const earliestRootCreatingTimeForReuse = rawManifest.maxRootReuseAgeInDays ?
     100            Date.now() - rawManifest.maxRootReuseAgeInDays * 24 * 3600 * 1000 : 0;
     101
    98102        for (const group of allTestGroupsInTask) {
    99103            if (group.id() == this.testGroupId())
    … …  
    108112                if (!buildRequest.commitSet().equalsIgnoringRoot(this.commitSet()))
    109113                    continue;
    110                 if (!buildRequest.commitSet().areAllRootsAvailable())
     114                if (!buildRequest.commitSet().areAllRootsAvailable(earliestRootCreatingTimeForReuse))
    111115                    continue;
    112116                if (buildRequest.hasCompleted())

  • trunk/Websites/perf.webkit.org/public/v3/models/commit-set.js

    r269871 r270607  
    7575    commits() { return  Array.from(this._repositoryToCommitMap.values()); }
    7676
    77     areAllRootsAvailable()
    78     {
    79         return this.allRootFiles().every(rootFile => !rootFile.deletedAt() || this.customRoots().find(rootFile));
     77    areAllRootsAvailable(earliestCreationTime)
     78    {
     79        return this.allRootFiles().every(rootFile => (!rootFile.deletedAt() || this.customRoots().find(rootFile))
     80            && rootFile.createdAt() >= earliestCreationTime);
    8081    }
    8182

  • trunk/Websites/perf.webkit.org/public/v3/models/manifest.js

    r269083 r270607  
    2323    }
    2424
    25     static fetch()
     25    static async fetch()
    2626    {
    2727        this.reset();
    28         return RemoteAPI.getJSON('/data/manifest.json').catch(function () {
    29             return RemoteAPI.getJSON('/api/manifest/');
    30         }).then(this._didFetchManifest.bind(this));
     28        const rawManifest = await this.fetchRawResponse();
     29        return this._didFetchManifest(rawManifest);
     30    }
     31
     32    static async fetchRawResponse()
     33    {
     34        try {
     35            return await RemoteAPI.getJSON('/data/manifest.json');
     36        } catch(error) {
     37            if (error != 404)
     38                throw `Failed to fetch manifest.json with ${error}`
     39            return await RemoteAPI.getJSON('/api/manifest/');
     40        }
    3141    }
    3242
    … …  
    92102            dashboards: rawResponse.dashboards, // FIXME: Add an abstraction around dashboards.
    93103            summaryPages: rawResponse.summaryPages,
     104            testAgeToleranceInHours: rawResponse.testAgeToleranceInHours,
     105            maxRootReuseAgeInDays: rawResponse.maxRootReuseAgeInDays,
    94106        }
    95107    }

  • trunk/Websites/perf.webkit.org/server-tests/api-manifest-tests.js

    r269083 r270607  
    1515        return TestServer.remoteAPI().getJSON('/api/manifest').then((manifest) => {
    1616            assert.deepEqual(Object.keys(manifest).sort(), ['all', 'bugTrackers', 'builders', 'dashboard', 'dashboards',
    17                 'fileUploadSizeLimit', 'metrics', 'platformGroups', 'repositories', 'siteTitle', 'status', 'summaryPages', 'testAgeToleranceInHours', 'tests', 'triggerables']);
     17                'fileUploadSizeLimit', 'maxRootReuseAgeInDays', 'metrics', 'platformGroups', 'repositories', 'siteTitle',
     18                'status', 'summaryPages', 'testAgeToleranceInHours', 'tests', 'triggerables']);
    1819
    1920            assert.deepStrictEqual(manifest, {
    … …  
    2526                dashboards: {},
    2627                fileUploadSizeLimit: 2097152, // 2MB during testing.
     28                maxRootReuseAgeInDays: null,
    2729                metrics: {},
    2830                platformGroups: {},

  • trunk/Websites/perf.webkit.org/unit-tests/build-request-tests.js

    r269871 r270607  
    22
    33const assert = require('assert');
     4const crypto = require('crypto');
    45
    56require('../tools/js/v3-models.js');
    … …  
    170171    if (!thirdTestGroupOverrides)
    171172        thirdTestGroupOverrides = {};
     173    const yesterday = Date.now() - 24 * 3600 * 1000;
    172174    return {
    173175        "testGroups": [{
    … …  
    370372        "commitSets": [{
    371373            "id": "4255",
    372             "revisionItems": [{"commit": "87832"}, {"commit": "93116"}],
     374            "revisionItems": [{"commit": "87832", rootFile: 101}, {"commit": "93116"}],
    373375            "customRoots": [],
    374376        }, {
    … …  
    398400            "time": 1448225325650
    399401        }],
    400         "uploadedFiles": [],
     402        "uploadedFiles": [{id: 101, filename: 'root-101', extension: '.tgz', size: 1,
     403            createdAt: yesterday, sha256: crypto.createHash('sha256').update('root-101').digest('hex')}],
    401404        "status": "OK"
    402405    };
    … …  
    427430            assert.equal(requests[0].method, 'GET');
    428431            requests[0].resolve(oneTestGroup());
     432
     433            await MockRemoteAPI.waitForRequest();
     434            assert.equal(requests.length, 2);
     435            assert.equal(requests[1].url, '/data/manifest.json');
     436            assert.equal(requests[1].method, 'GET');
     437            requests[1].resolve({maxRootReuseAgeInDays: 30});
    429438
    430439            const result = await promise;
    … …  
    449458            requests[0].resolve(threeTestGroups(overrides));
    450459
     460            await MockRemoteAPI.waitForRequest();
     461            assert.equal(requests.length, 2);
     462            assert.equal(requests[1].url, '/data/manifest.json');
     463            assert.equal(requests[1].method, 'GET');
     464            requests[1].resolve({maxRootReuseAgeInDays: 30});
     465
    451466            const result = await promise;
    452467            assert.equal(result, BuildRequest.findById(16989))
     468        });
     469
     470        it('should not reuse a root that is older than "maxReuseRootAge"', async () => {
     471            const overrides = {
     472                task: '1376',
     473                platform: '32',
     474                status: ['completed', 'pending', 'pending', 'pending']
     475            }
     476            const data = threeTestGroups(overrides);
     477
     478            const platformId = data.buildRequests[0].platform;
     479            Platform.ensureSingleton(platformId, {id: platformId, metrics: [], name: 'some platform'});
     480            const request = BuildRequest.constructBuildRequestsFromData(data)[0];
     481            const promise = request.findBuildRequestWithSameRoots();
     482            assert.equal(requests.length, 1);
     483
     484            assert.equal(requests[0].url, '/api/test-groups?task=1376');
     485            assert.equal(requests[0].method, 'GET');
     486            requests[0].resolve(threeTestGroups(overrides));
     487
     488            await MockRemoteAPI.waitForRequest();
     489            assert.equal(requests.length, 2);
     490            assert.equal(requests[1].url, '/data/manifest.json');
     491            assert.equal(requests[1].method, 'GET');
     492            requests[1].resolve({maxRootReuseAgeInDays: 0.5});
     493
     494            const result = await promise;
     495            assert.equal(result, null)
    453496        });
    454497
    … …  
    470513            requests[0].resolve(threeTestGroups(overrides));
    471514
     515            await MockRemoteAPI.waitForRequest();
     516            assert.equal(requests.length, 2);
     517            assert.equal(requests[1].url, '/data/manifest.json');
     518            assert.equal(requests[1].method, 'GET');
     519            requests[1].resolve({maxRootReuseAgeInDays: 30});
     520
    472521            let result = await promise;
    473522            assert.equal(result, BuildRequest.findById(16989))
    … …  
    480529            assert.equal(requests[0].method, 'GET');
    481530            requests[0].resolve(threeTestGroups(overrides));
     531
     532            await MockRemoteAPI.waitForRequest();
     533            assert.equal(requests.length, 2);
     534            assert.equal(requests[1].url, '/data/manifest.json');
     535            assert.equal(requests[1].method, 'GET');
     536            requests[1].resolve({maxRootReuseAgeInDays: 30});
    482537
    483538            result = await promise;
    … …  
    502557            assert.equal(requests[0].method, 'GET');
    503558            requests[0].resolve(threeTestGroups(overrides));
     559
     560            await MockRemoteAPI.waitForRequest();
     561            assert.equal(requests.length, 2);
     562            assert.equal(requests[1].url, '/data/manifest.json');
     563            assert.equal(requests[1].method, 'GET');
     564            requests[1].resolve({maxRootReuseAgeInDays: 30});
    504565
    505566            const result = await promise;
    … …  
    525586            assert.equal(requests[0].method, 'GET');
    526587            requests[0].resolve(threeTestGroups(overrides));
     588
     589            await MockRemoteAPI.waitForRequest();
     590            assert.equal(requests.length, 2);
     591            assert.equal(requests[1].url, '/data/manifest.json');
     592            assert.equal(requests[1].method, 'GET');
     593            requests[1].resolve({maxRootReuseAgeInDays: 30});
    527594
    528595            const result = await promise;
    … …  
    552619            requests[0].resolve(threeTestGroups(secondOverrides, thirdOverrides));
    553620
     621            await MockRemoteAPI.waitForRequest();
     622            assert.equal(requests.length, 2);
     623            assert.equal(requests[1].url, '/data/manifest.json');
     624            assert.equal(requests[1].method, 'GET');
     625            requests[1].resolve({maxRootReuseAgeInDays: 30});
     626
    554627            const result = await promise;
    555628            assert.equal(result, BuildRequest.findById(16993))
    … …  
    578651            requests[0].resolve(threeTestGroups(secondOverrides, thirdOverrides));
    579652
     653            await MockRemoteAPI.waitForRequest();
     654            assert.equal(requests.length, 2);
     655            assert.equal(requests[1].url, '/data/manifest.json');
     656            assert.equal(requests[1].method, 'GET');
     657            requests[1].resolve({maxRootReuseAgeInDays: 30});
     658
    580659            const result = await promise;
    581660            assert.equal(result, BuildRequest.findById(16993));
Note: See TracChangeset for help on using the changeset viewer.