Changeset 252490 in webkit


Ignore:
Timestamp:
Nov 15, 2019 11:18:52 AM (4 years ago)
Author:
Jonathan Bedard
Message:

results.webkit.org: Report JSC tests to the results database
​https://bugs.webkit.org/show_bug.cgi?id=204091
<rdar://problem/49778900>

Reviewed by Aakash Jain.

Upload results to the specified results database.

DEMONSTRATION OF FAILURE IN EWS BEHAVIOR, DO NOT LAND!

  • Scripts/run-javascriptcore-tests:

(runTest): Parse binary output to mark individual tests as passing
or failing.
(uploadConfiguration): Return a hash map representing the configuration
For this specific test run.
(uploadResults): If a report URL is defined, upload test results to that URL.

  • Scripts/webkitdirs.pm:

(splitVersionString): splitVersionString is supposed to be a generally used
function, so should not set a global variable.
(determineOSXVersion): Remove newline from version number.

Location:
trunk/Tools
Files:
3 edited

Legend:

Unmodified
Added
Removed

  • trunk/Tools/ChangeLog

    r252489 r252490  
     12019-11-15  Jonathan Bedard  <jbedard@apple.com>
     2
     3        results.webkit.org: Report JSC tests to the results database
     4        https://bugs.webkit.org/show_bug.cgi?id=204091
     5        <rdar://problem/49778900>
     6
     7        Reviewed by Aakash Jain.
     8
     9        Upload results to the specified results database.
     10
     11        DEMONSTRATION OF FAILURE IN EWS BEHAVIOR, DO NOT LAND!
     12
     13        * Scripts/run-javascriptcore-tests:
     14        (runTest): Parse binary output to mark individual tests as passing
     15        or failing.
     16        (uploadConfiguration): Return a hash map representing the configuration
     17        For this specific test run.
     18        (uploadResults): If a report URL is defined, upload test results to that URL.
     19        * Scripts/webkitdirs.pm:
     20        (splitVersionString): splitVersionString is supposed to be a generally used
     21        function, so should not set a global variable.
     22        (determineOSXVersion): Remove newline from version number.
     23
    1242019-11-15  Jonathan Bedard  <jbedard@apple.com>
    225

  • trunk/Tools/Scripts/run-javascriptcore-tests

    r251161 r252490  
    5858my $memoryLimited;
    5959
     60my $report;
     61my $buildbotMaster;
     62my $builderName;
     63my $buildNumber;
     64my $buildbotWorker;
     65
    6066my $buildJSC = 1;
    6167my $copyJSC = 1;
    … …  
    9298my $remoteHost = 0;
    9399my $model = 0;
     100my $version;
     101my $versionName;
     102my $sdk;
    94103my $failFast = 1;
    95104my %jsonData = ();
     105my %reportData = ();
    96106my @testResults = ();
    97107my $isTestFailed = 0;
    … …  
    232242  --remote-config-file=         Same as remote, but read config from JSON file.
    233243  --model=                      Specify remote hardware model, this info used for determine what jsc tests should run on remote
     244  --version                     Specify the version number of the device running tests.
     245  --version-name                Specify the version name of the hardware running tests.
     246  --sdk                         Specific SDK or OS version of the form ##*###
    234247  --extra-tests=                Path to a file containing extra tests
    235248  --child-processes=            Specify the number of child processes.
    … …  
    252265  --gmalloc:                    Run tests with Guard Malloc enabled (if no path is given: $gmallocDefaultPath is used)
    253266  --verbose:                    Verbose output (specify more than once to increase verbosity).
     267
     268  --report:                     Results database url to report results to.
     269  --buildbot-master:            The url of the buildbot master.
     270  --builder-name:               The name of the buildbot builder tests were run on.
     271  --build-number:               The buildbot build number tests are associated with.
     272  --buildbot-worker:            The buildbot worker tests were run on.
    254273
    255274Environment Variables:
    … …  
    290309    'remote=s' => \$remoteHost,
    291310    'model=s' => \$model,
     311    'version=s' => \$version,
     312    'version-name=s' => \$versionName,
     313    'sdk=s' => \$sdk,
    292314    'remote-config-file=s' => \$remoteConfigFile,
    293315    'child-processes=s' => \$childProcesses,
    … …  
    301323    'env-vars=s' => \$envVars,
    302324    'gmalloc:s' => \$gmallocPath,
    303     'verbose+' => \$verbose
     325    'verbose+' => \$verbose,
     326    'report=s' => \$report,
     327    'buildbot-master=s' => \$buildbotMaster,
     328    'builder-name=s' => \$builderName,
     329    'build-number=s' => \$buildNumber,
     330    'buildbot-worker=s' => \$buildbotWorker,
    304331);
    305332
    … …  
    316343}
    317344
     345if ($version) {
     346    $version = splitVersionString($version);
     347}
     348
    318349sub enableTestOrNot {
    319350    my ($state) = @_;
    … …  
    324355    }
    325356    return $state;
     357}
     358
     359sub configurationForUpload()
     360{
     361    my $platform;
     362    my $sdk;
     363    my $simulator = 0;
     364
     365    if (index(xcodeSDKPlatformName(), "simulator") != -1) {
     366        $simulator = 1;
     367    }
     368
     369    if (index($model, 'iPhone') != -1 || index($model, 'iPad') != -1) {
     370        $platform = 'ios';
     371        if (!$version) {
     372            $version = iosVersion();
     373        }
     374    } elsif (index($model, 'watch') != -1 || index(xcodeSDKPlatformName(), "watch") != -1) {
     375        $platform = 'watchos';
     376        die "No watchOS version specified" if !$version;
     377    } elsif (index(xcodeSDKPlatformName(), "appletv") != -1) {
     378        $platform = 'tvos';
     379        die "No tvOS version specified" if !$version;
     380    } elsif (isGtk()) {
     381        $platform = 'GTK';
     382        if (!$version) {
     383            chomp($version = `/usr/bin/uname -r`);
     384            $version = splitVersionString($version);
     385        }
     386    } elsif (isWPE()) {
     387        $platform = 'WPE';
     388        if (!$version) {
     389            chomp($version = `/usr/bin/uname -r`);
     390            $version = splitVersionString($version);
     391        }
     392    } elsif (isAnyWindows()) {
     393        $platform = 'win';
     394        if (!$version) {
     395            $version = winVersion();
     396        }
     397    } elsif (isAppleMacWebKit()) {
     398        $platform = 'mac';
     399        chomp($model = `/usr/sbin/sysctl -n hw.model`);
     400        if (!$version) {
     401            $version = osXVersion();
     402        }
     403        if (!$sdk) {
     404            chomp($sdk = `/usr/bin/sw_vers -buildVersion`);
     405        }
     406
     407        if (!$versionName) {
     408            if ($version->{minor} eq 15) {
     409                $versionName = "Catalina";
     410            } elsif ($version->{minor} eq 14) {
     411                $versionName = "Mojave";
     412            } elsif ($version->{minor} eq 13) {
     413                $versionName = "High Sierra";
     414            }
     415        }
     416    } else {
     417        die "Cannot determine platform\n";
     418    }
     419
     420    my $result = {
     421        platform => $platform,
     422        architecture => architecture(),
     423        is_simulator => $simulator,
     424        style => lc(configuration()),
     425        version => "$version->{major}.$version->{minor}.$version->{subminor}",
     426    };
     427    if ($model) {
     428        $result->{model} = $model;
     429    }
     430    if ($versionName) {
     431        $result->{version_name} = $versionName;
     432    }
     433    if ($sdk) {
     434        $result->{sdk} = $sdk;
     435    }
     436    return $result;
    326437}
    327438
    … …  
    343454
    344455setConfigurationProductDir(Cwd::abs_path($root)) if (defined($root));
     456configurationForUpload() if (defined($report));
    345457
    346458if (defined($jsonFileName)) {
    … …  
    375487setPathForRunningWebKitApp(\%ENV) if isCygwin();
    376488
     489my $startTime = time();
     490my $endTime = $startTime;
     491
    377492sub testPath {
    378493    my ($productDir, $testName) = @_;
    … …  
    398513    }
    399514
    400     # Use an "indirect object" so that system() won't get confused if the path
    401     # contains spaces (see perldoc -f exec).
    402     my $testResult = system { $command[0] } @command;
     515    my $testResult = 0;
     516    my $failureFlag = 0;
     517    my $lastOptimizeLevel;
     518
     519    open(TEST, "-|", "@command 2>&1") or die "Failed to run @command";
     520    while ( my $line = <TEST> ) {
     521        my $subTestName;
     522        print $line;
     523
     524        # e.g.: 'O2: testRotRWithImmShift(uint64-max, uint32-min): OK!''
     525        if ($line =~ /(O\d): (\S+)(\(.*\)): ([A-Za-z]+)!\n/) {
     526            $lastOptimizeLevel = $1;
     527            $subTestName = "$testName.$1.$2$3";
     528            $failureFlag += 1 if ($4 ne "OK");
     529        }
     530        # Assertion in binary
     531        # e.g.: ASSERTION FAILED: things[0] == 2
     532        elsif ($line =~ /ASSERTION FAILED:/) {
     533            $failureFlag += 1;
     534            next;
     535        }
     536        # e.g.: file/path(397) : void (anonymous namespace)::testShuffleSimpleBroadcast()
     537        elsif ($failureFlag && $line =~ /.*\(\d+\) : \S+ .*::(\S+)(\(.*\))/) {
     538            if (defined $lastOptimizeLevel) {
     539                $subTestName = "$testName.$lastOptimizeLevel.$1$2";
     540            } else {
     541                $subTestName = "$testName.$1$2";
     542            }
     543        }
     544        # sub-test from the c-api, of the form:
     545        # e.g.: 'testapi[15422:16904790] parallelPromiseResolveTest(): done.''
     546        elsif ($line =~ /\[\d+:\d+\] (\S+)(): done\.\n/) {
     547            $subTestName = "$testName.objective-c.$1";
     548        }
     549        # End of C-API tests (meaning we just finished the ungrouped main tests)
     550        elsif ($line =~ /Starting C-API tests in C++/) {
     551            $subTestName = "$testName.objective-c.main";
     552        }
     553        # End of testapi
     554        # e.g.: FAIL: Some tests failed.
     555        elsif ($line =~ /FAIL: Some tests failed\./) {
     556            $failureFlag = 0;
     557            next;
     558        }
     559        # Test cases inside API tests
     560        # e.g.: 'testapi[15422:16904790] TEST: "2 + 2": PASSED'
     561        elsif ($line =~ /\[\d+:\d+\] TEST: ".*": ([A-Z]+)\n/) {
     562            $failureFlag += 1 if ($1 ne "PASSED" && $1 ne "PASS");
     563            next;
     564        }
     565        # e.g.: 'testBitXorAndAndArgs(0, 0,uint32-max): OK!''
     566        elsif ($line =~ /(\S+)(\(.*\)): ([A-Za-z]+)!\n/) {
     567            $subTestName = "$testName.$1$2";
     568            $failureFlag += 1 if ($3 ne "OK");
     569        }
     570        # General testcase statement, filtered to exclude some junk logging
     571        # e.g.: 'PASS: DFG script timed out as expected when no callback is specified.''
     572        elsif ($line =~ /([A-Z]+): (.*)\.*\n/ && !($line =~ /[A-Z]+: \d+\n/ || $line =~ /\[\d+:\d+\] [A-Z]+:/)) {
     573            $subTestName = "$testName.$2";
     574            $failureFlag += 1 if ($1 ne "PASS");
     575        }
     576        # e.g.: 'test description: PASSED'
     577        elsif ($line =~ /.*: ([A-Z]+)\n/) {
     578            $failureFlag += 1 if ($1 ne "PASSED" && $1 ne "PASS");
     579            next;
     580        } else {
     581            next;
     582        }
     583
     584        if (!defined $reportData{$subTestName}) {
     585            print("$subTestName $failureFlag\n") if $failureFlag;
     586            $reportData{$subTestName} = $failureFlag ? {actual => "FAIL"} : {actual => "PASS"};
     587        }
     588        $failureFlag = 0;
     589    }
     590    $testResult = close(TEST) ? 0 : $?;
     591
    403592    my $exitStatus = exitStatus($testResult);
    404593    print "$testName completed with rc=$testResult ($exitStatus)\n\n";
    … …  
    416605        reportTestFailures();
    417606        writeJsonDataIfApplicable();
    418         exit exitStatus($testResult);
     607
     608        $endTime = time();
     609        uploadResults();
     610        exit $exitStatus;
    419611    }
    420612}
    … …  
    447639chdirWebKit();
    448640
     641# FIXME: report stress test failures https://bugs.webkit.org/show_bug.cgi?id=204096
    449642runJSCStressTests();
    450643reportTestFailures();
     644
     645$endTime = time();
     646uploadResults();
    451647
    452648if ($isTestFailed) {
    … …  
    667863    }
    668864}
     865
     866sub uploadResults
     867{
     868    if (not defined $report) {
     869        print "Skipping upload to results database since no report URL was specified\n";
     870        return 0;
     871    }
     872
     873    my %upload = (
     874        version => 0,
     875        suite => 'javascriptcore-tests',
     876        # FIXME: Integrate branches, https://bugs.webkit.org/show_bug.cgi?id=204094
     877        # FIXME: Integrate the Safari repository, https://bugs.webkit.org/show_bug.cgi?id=204095
     878        commits => [{
     879            repository_id => 'webkit',
     880            id => determineCurrentSVNRevision(),
     881        }],
     882        configuration => configurationForUpload(),
     883        test_results => {
     884            run_stats => {
     885                start_time => $startTime,
     886                end_time => $endTime,
     887                tests_skipped => 0,
     888            },
     889            results => \%reportData,
     890        },
     891        timestamp => $startTime,
     892    );
     893    if (defined $buildbotMaster and defined $builderName and defined $buildNumber and defined $buildbotWorker) {
     894        $upload{test_results}{details} = {
     895            buildbot_master => $buildbotMaster,
     896            builder_name => $builderName,
     897            build_number => $buildNumber,
     898            buildbot_worker => $buildbotWorker,
     899        };
     900    } else {
     901        print "    No buildbot details provided, test run will not be linked to CI system\n";
     902    }
     903    if (defined $ENV{'RESULTS_SERVER_API_KEY'}) {
     904        $upload{'api_key'} = $ENV{'RESULTS_SERVER_API_KEY'};
     905    }
     906
     907    print "Uploading results to $report\n";
     908
     909    open(HANDLE, "|-", "curl -X POST $report/api/upload -H 'Content-Type: application/json' -f -d '\@-'") or die "Failed to open curl";
     910
     911    # Json conforming to https://results.webkit.org/documentation#API-Uploads.
     912    my $encodedUpload = encode_json(\%upload);
     913    print HANDLE "$encodedUpload\n\0";
     914    my $success = close HANDLE;
     915
     916    if ($success) {
     917        print "Upload successful!\n";
     918        return 0;
     919    }
     920    print "Upload to $report failed\n";
     921    return 1;
     922}

  • trunk/Tools/Scripts/webkitdirs.pm

    r251293 r252490  
    15641564    my @splitVersion = split(/\./, $versionString);
    15651565    @splitVersion >= 2 or die "Invalid version $versionString";
    1566     $osXVersion = {
    1567             "major" => $splitVersion[0],
    1568             "minor" => $splitVersion[1],
    1569             "subminor" => (defined($splitVersion[2]) ? $splitVersion[2] : 0),
     1566    return {
     1567        "major" => $splitVersion[0],
     1568        "minor" => $splitVersion[1],
     1569        "subminor" => (defined($splitVersion[2]) ? $splitVersion[2] : 0),
    15701570    };
    15711571}
    … …  
    15801580    }
    15811581
    1582     my $versionString = `sw_vers -productVersion`;
     1582    chomp(my $versionString = `sw_vers -productVersion`);
    15831583    $osXVersion = splitVersionString($versionString);
    15841584}
Note: See TracChangeset for help on using the changeset viewer.