Attachment #210833 for bug #120918

View Details Raw Unified Return to Bug 120918



2013-09-06  Filip Pizlo  <fpizlo@apple.com>

        It should be possible to run-javascriptcore-tests with only a subset of the test suites
        https://bugs.webkit.org/show_bug.cgi?id=120918

        Reviewed by NOBODY (OOPS!).

        * Scripts/run-javascriptcore-tests:

2013-09-06  Filip Pizlo  <fpizlo@apple.com>

        The 'failed' file should report collectionName/testName rather than just




my $buildJSC = 1;

my $runTestAPI = isAppleMacWebKit() || isAppleWinWebKit();

my $runMozilla = 1;

# FIXME: run-fast-jsc tests doesn't work properly on Windows
# https://bugs.webkit.org/show_bug.cgi?id=120765
my $runFastJS = !isAppleWinWebKit();

# FIXME: run-jsc-stress-tests should be ported to other platforms.
# https://bugs.webkit.org/show_bug.cgi?id=120809
my $runJSCStress = isAppleMacWebKit();

my $programName = basename($0);
my $buildJSCDefault = $buildJSC ? "will check" : "will not check";
my $testapiDefault = $runTestAPI ? "will run" : "will not run";
my $mozillaDefault = $runMozilla ? "will run" : "will not run";
my $fastJSDefault = $runFastJS ? "will run" : "will not run";
my $jscStressDefault = $runJSCStress ? "will run" : " will not run";
my $usage = <<EOF;
Usage: $programName [options] [options to pass to build system]
  --help                        Show this help message
  --jsDriver-args=              A string of arguments to pass to jsDriver.pl
  --root=                       Path to pre-built root containing jsc
  --[no-]build                  Check (or don't check) to see if the jsc build is up-to-date (default: $buildJSCDefault)
  --[no-]testapi                Run (or don't run) testapi (default: $testapiDefault)
  --[no-]mozilla                Run (or don't run) the Mozilla tests (default: $mozillaDefault)
  --[no-]fast-js                Run (or don't run) the fast/js tests (default: $fastJSDefault)
  --[no-]jsc-stress             Run (or don't run) the JSC stress tests (default: $jscStressDefault)
EOF

GetOptions(
    'j|jsDriver-args=s' => \$jsDriverArgs,
    'root=s' => \$root,
    'build!' => \$buildJSC,
    'testapi!' => \$runTestAPI,
    'mozilla!' => \$runMozilla,
    'fast-js!' => \$runFastJS,
    'jsc-stress!' => \$runJSCStress,
    'help' => \$showHelp
);


}

#run api tests
if ($runTestAPI) {
    chdirWebKit();
    chdir($productDir) or die "Failed to switch directory to '$productDir'\n";
    my @command = (testapiPath($productDir));


# Find JavaScriptCore directory
chdirWebKit();
chdir("Source/JavaScriptCore");
chdir "tests/mozilla" or die "Failed to switch directory to 'tests/mozilla'\n";
my @jsMozillaDriverCmd = ("perl", "jsDriver.pl", "-e", "squirrelfish", "-s", jscPath($productDir), "-f", "actual.html", @jsArgs);
if (isGtk() || isEfl()) {
    my @jhbuildPrefix = sourceDir() . "/Tools/jhbuild/jhbuild-wrapper";

    if (isEfl()) {
        push(@jhbuildPrefix, '--efl');
    } elsif (isGtk()) {
        push(@jhbuildPrefix, '--gtk');
    }
    push(@jhbuildPrefix, 'run');

    unshift(@jsMozillaDriverCmd, @jhbuildPrefix);
} elsif (isIOSWebKit() && willUseIOSSimulatorSDKWhenBuilding()) {
    push @jsMozillaDriverCmd, ("--sdk", xcodeSDK());
}
print "Running: " . join(" ", @jsMozillaDriverCmd) . "\n";
my $result = system(@jsMozillaDriverCmd);
exit exitStatus($result)  if $result;

my %mozillaFailures;

open EXPECTED, "expected.html" or die "Failed to open 'expected.html'\n";
while (<EXPECTED>) {
    last if /failures reported\.$/;
}
while (<EXPECTED>) {
    chomp;
    $mozillaFailures{$_} = 1;
}
close EXPECTED;

my %newMozillaFailures;

if ($runMozilla) {
    chdir("Source/JavaScriptCore");
    chdir "tests/mozilla" or die "Failed to switch directory to 'tests/mozilla'\n";
    my @jsMozillaDriverCmd = ("perl", "jsDriver.pl", "-e", "squirrelfish", "-s", jscPath($productDir), "-f", "actual.html", @jsArgs);
    if (isGtk() || isEfl()) {
        my @jhbuildPrefix = sourceDir() . "/Tools/jhbuild/jhbuild-wrapper";
    
        if (isEfl()) {
            push(@jhbuildPrefix, '--efl');
        } elsif (isGtk()) {
            push(@jhbuildPrefix, '--gtk');
        }
        push(@jhbuildPrefix, 'run');
    
        unshift(@jsMozillaDriverCmd, @jhbuildPrefix);
    } elsif (isIOSWebKit() && willUseIOSSimulatorSDKWhenBuilding()) {
        push @jsMozillaDriverCmd, ("--sdk", xcodeSDK());
    }
    print "Running: " . join(" ", @jsMozillaDriverCmd) . "\n";
    my $result = system(@jsMozillaDriverCmd);
    exit exitStatus($result)  if $result;
    
    open EXPECTED, "expected.html" or die "Failed to open 'expected.html'\n";
    while (<EXPECTED>) {
        last if /failures reported\.$/;
    }
    while (<EXPECTED>) {
        chomp;
        $mozillaFailures{$_} = 1;
    }
    close EXPECTED;
    
    open ACTUAL, "actual.html" or die "Failed to open 'actual.html'";
    while (<ACTUAL>) {
        last if /failures reported\.$/;
    }
    while (<ACTUAL>) {
        chomp;
        if ($mozillaFailures{$_}) {
            delete $mozillaFailures{$_};
        } else {
            $newMozillaFailures{$_} = 1;
        }
    }
    close ACTUAL;
}
close ACTUAL;

chdirWebKit();
my $fastJSResultsDir = $productDir . "/fast-jsc-results";

# FIXME: run-fast-jsc tests doesn't work properly on Windows
# https://bugs.webkit.org/show_bug.cgi?id=120765
my $runFastJS = !isAppleWinWebKit();

if ($runFastJS) {
    # Run the fast/js tests.
    my @fastJSDriverCmd = ("/bin/sh", "Tools/Scripts/run-fast-jsc", "-j", jscPath($productDir), "-r", $fastJSResultsDir, "-t", "LayoutTests");
    print "Running: " . join(" ", @fastJSDriverCmd) . "\n";
    my $result = system(@fastJSDriverCmd);
    exit exitStatus($result) if $result;
}

my $jscStressResultsDir = $productDir . "/jsc-stress-results";

# FIXME: run-jsc-stress-tests should be ported to other platforms.
# https://bugs.webkit.org/show_bug.cgi?id=120809
my $runJSCStress = isAppleMacWebKit();
if ($runJSCStress) {
    my @jscStressDriverCmd = (
        "/usr/bin/env", "ruby", "Tools/Scripts/run-jsc-stress-tests",

        "LayoutTests/fast/js/regress/script-tests"
    );
    print "Running: " . join(" ", @jscStressDriverCmd) . "\n";
    my $result = system(@jscStressDriverCmd);
    exit exitStatus($result) if $result;
}



print "\n";

if ($runMozilla) {
    print "Results for Mozilla tests:\n";
    printThingsFound($numNewMozillaFailures, "regression", "regressions", "found");
    printThingsFound($numOldMozillaFailures, "test", "tests", "fixed");
    print "    OK.\n" if $numNewMozillaFailures == 0;
    
    print "\n";
}

if ($runFastJS) {
    print "Results for fast/js tests:\n";

    print "\n";
}

exit(1)  if ($runMozilla && $numNewMozillaFailures) || ($runFastJS && ($numJSFailures || $numJSCrashes)) || ($runJSCStress && $numJSCStressFailures);

Return to Bug 120918

Lines 68-87 my $showHelp; Tools/Scripts/run-javascriptcore-tests_sec1
- Tools/Scripts/run-javascriptcore-tests -59 / +80 lines
68		68
69	my $buildJSC = 1;	69	my $buildJSC = 1;
70		70
		71	my $runTestAPI = isAppleMacWebKit() \|\| isAppleWinWebKit();
		72
		73	my $runMozilla = 1;
		74
		75	# FIXME: run-fast-jsc tests doesn't work properly on Windows
		76	# https://bugs.webkit.org/show_bug.cgi?id=120765
		77	my $runFastJS = !isAppleWinWebKit();
		78
		79	# FIXME: run-jsc-stress-tests should be ported to other platforms.
		80	# https://bugs.webkit.org/show_bug.cgi?id=120809
		81	my $runJSCStress = isAppleMacWebKit();
		82
71	my $programName = basename($0);	83	my $programName = basename($0);
72	my $buildJSCDefault = $buildJSC ? "will check" : "will not check";	84	my $buildJSCDefault = $buildJSC ? "will check" : "will not check";
		85	my $testapiDefault = $runTestAPI ? "will run" : "will not run";
		86	my $mozillaDefault = $runMozilla ? "will run" : "will not run";
		87	my $fastJSDefault = $runFastJS ? "will run" : "will not run";
		88	my $jscStressDefault = $runJSCStress ? "will run" : " will not run";
73	my $usage = <<EOF;	89	my $usage = <<EOF;
74	Usage: $programName [options] [options to pass to build system]	90	Usage: $programName [options] [options to pass to build system]
75	--help Show this help message	91	--help Show this help message
76	--jsDriver-args= A string of arguments to pass to jsDriver.pl	92	--jsDriver-args= A string of arguments to pass to jsDriver.pl
77	--root= Path to pre-built root containing jsc	93	--root= Path to pre-built root containing jsc
78	--[no-]build Check (or don't check) to see if the jsc build is up-to-date (default: $buildJSCDefault)	94	--[no-]build Check (or don't check) to see if the jsc build is up-to-date (default: $buildJSCDefault)
		95	--[no-]testapi Run (or don't run) testapi (default: $testapiDefault)
		96	--[no-]mozilla Run (or don't run) the Mozilla tests (default: $mozillaDefault)
		97	--[no-]fast-js Run (or don't run) the fast/js tests (default: $fastJSDefault)
		98	--[no-]jsc-stress Run (or don't run) the JSC stress tests (default: $jscStressDefault)
79	EOF	99	EOF
80		100
81	GetOptions(	101	GetOptions(
82	'j\|jsDriver-args=s' => \$jsDriverArgs,	102	'j\|jsDriver-args=s' => \$jsDriverArgs,
83	'root=s' => \$root,	103	'root=s' => \$root,
84	'build!' => \$buildJSC,	104	'build!' => \$buildJSC,
		105	'testapi!' => \$runTestAPI,
		106	'mozilla!' => \$runMozilla,
		107	'fast-js!' => \$runFastJS,
		108	'jsc-stress!' => \$runJSCStress,
85	'help' => \$showHelp	109	'help' => \$showHelp
86	);	110	);
87		111
Lines 126-132 sub testapiPath($) Tools/Scripts/run-javascriptcore-tests_sec2
126	}	150	}
127		151
128	#run api tests	152	#run api tests
129	if (isAppleMacWebKit() \|\| isAppleWinWebKit()) {	153	if ($runTestAPI) {
130	chdirWebKit();	154	chdirWebKit();
131	chdir($productDir) or die "Failed to switch directory to '$productDir'\n";	155	chdir($productDir) or die "Failed to switch directory to '$productDir'\n";
132	my @command = (testapiPath($productDir));	156	my @command = (testapiPath($productDir));
Lines 140-214 if (isAppleMacWebKit() \|\| isAppleWinWebK Tools/Scripts/run-javascriptcore-tests_sec3
140		164
141	# Find JavaScriptCore directory	165	# Find JavaScriptCore directory
142	chdirWebKit();	166	chdirWebKit();
143	chdir("Source/JavaScriptCore");
144	chdir "tests/mozilla" or die "Failed to switch directory to 'tests/mozilla'\n";
145	my @jsMozillaDriverCmd = ("perl", "jsDriver.pl", "-e", "squirrelfish", "-s", jscPath($productDir), "-f", "actual.html", @jsArgs);
146	if (isGtk() \|\| isEfl()) {
147	my @jhbuildPrefix = sourceDir() . "/Tools/jhbuild/jhbuild-wrapper";
148
149	if (isEfl()) {
150	push(@jhbuildPrefix, '--efl');
151	} elsif (isGtk()) {
152	push(@jhbuildPrefix, '--gtk');
153	}
154	push(@jhbuildPrefix, 'run');
155
156	unshift(@jsMozillaDriverCmd, @jhbuildPrefix);
157	} elsif (isIOSWebKit() && willUseIOSSimulatorSDKWhenBuilding()) {
158	push @jsMozillaDriverCmd, ("--sdk", xcodeSDK());
159	}
160	print "Running: " . join(" ", @jsMozillaDriverCmd) . "\n";
161	my $result = system(@jsMozillaDriverCmd);
162	exit exitStatus($result) if $result;
163		167
164	my %mozillaFailures;	168	my %mozillaFailures;
165
166	open EXPECTED, "expected.html" or die "Failed to open 'expected.html'\n";
167	while (<EXPECTED>) {
168	last if /failures reported\.$/;
169	}
170	while (<EXPECTED>) {
171	chomp;
172	$mozillaFailures{$_} = 1;
173	}
174	close EXPECTED;
175
176	my %newMozillaFailures;	169	my %newMozillaFailures;
177		170
178	open ACTUAL, "actual.html" or die "Failed to open 'actual.html'";	171	if ($runMozilla) {
179	while (<ACTUAL>) {	172	chdir("Source/JavaScriptCore");
180	last if /failures reported\.$/;	173	chdir "tests/mozilla" or die "Failed to switch directory to 'tests/mozilla'\n";
181	}	174	my @jsMozillaDriverCmd = ("perl", "jsDriver.pl", "-e", "squirrelfish", "-s", jscPath($productDir), "-f", "actual.html", @jsArgs);
182	while (<ACTUAL>) {	175	if (isGtk() \|\| isEfl()) {
183	chomp;	176	my @jhbuildPrefix = sourceDir() . "/Tools/jhbuild/jhbuild-wrapper";
184	if ($mozillaFailures{$_}) {	177
185	delete $mozillaFailures{$_};	178	if (isEfl()) {
186	} else {	179	push(@jhbuildPrefix, '--efl');
187	$newMozillaFailures{$_} = 1;	180	} elsif (isGtk()) {
		181	push(@jhbuildPrefix, '--gtk');
		182	}
		183	push(@jhbuildPrefix, 'run');
		184
		185	unshift(@jsMozillaDriverCmd, @jhbuildPrefix);
		186	} elsif (isIOSWebKit() && willUseIOSSimulatorSDKWhenBuilding()) {
		187	push @jsMozillaDriverCmd, ("--sdk", xcodeSDK());
		188	}
		189	print "Running: " . join(" ", @jsMozillaDriverCmd) . "\n";
		190	my $result = system(@jsMozillaDriverCmd);
		191	exit exitStatus($result) if $result;
		192
		193	open EXPECTED, "expected.html" or die "Failed to open 'expected.html'\n";
		194	while (<EXPECTED>) {
		195	last if /failures reported\.$/;
		196	}
		197	while (<EXPECTED>) {
		198	chomp;
		199	$mozillaFailures{$_} = 1;
188	}	200	}
		201	close EXPECTED;
		202
		203	open ACTUAL, "actual.html" or die "Failed to open 'actual.html'";
		204	while (<ACTUAL>) {
		205	last if /failures reported\.$/;
		206	}
		207	while (<ACTUAL>) {
		208	chomp;
		209	if ($mozillaFailures{$_}) {
		210	delete $mozillaFailures{$_};
		211	} else {
		212	$newMozillaFailures{$_} = 1;
		213	}
		214	}
		215	close ACTUAL;
189	}	216	}
190	close ACTUAL;
191		217
192	chdirWebKit();	218	chdirWebKit();
193	my $fastJSResultsDir = $productDir . "/fast-jsc-results";	219	my $fastJSResultsDir = $productDir . "/fast-jsc-results";
194		220
195	# FIXME: run-fast-jsc tests doesn't work properly on Windows
196	# https://bugs.webkit.org/show_bug.cgi?id=120765
197	my $runFastJS = !isAppleWinWebKit();
198
199	if ($runFastJS) {	221	if ($runFastJS) {
200	# Run the fast/js tests.	222	# Run the fast/js tests.
201	my @fastJSDriverCmd = ("/bin/sh", "Tools/Scripts/run-fast-jsc", "-j", jscPath($productDir), "-r", $fastJSResultsDir, "-t", "LayoutTests");	223	my @fastJSDriverCmd = ("/bin/sh", "Tools/Scripts/run-fast-jsc", "-j", jscPath($productDir), "-r", $fastJSResultsDir, "-t", "LayoutTests");
202	print "Running: " . join(" ", @fastJSDriverCmd) . "\n";	224	print "Running: " . join(" ", @fastJSDriverCmd) . "\n";
203	$result = system(@fastJSDriverCmd);	225	my $result = system(@fastJSDriverCmd);
204	exit exitStatus($result) if $result;	226	exit exitStatus($result) if $result;
205	}	227	}
206		228
207	my $jscStressResultsDir = $productDir . "/jsc-stress-results";	229	my $jscStressResultsDir = $productDir . "/jsc-stress-results";
208		230
209	# FIXME: run-jsc-stress-tests should be ported to other platforms.
210	# https://bugs.webkit.org/show_bug.cgi?id=120809
211	my $runJSCStress = isAppleMacWebKit();
212	if ($runJSCStress) {	231	if ($runJSCStress) {
213	my @jscStressDriverCmd = (	232	my @jscStressDriverCmd = (
214	"/usr/bin/env", "ruby", "Tools/Scripts/run-jsc-stress-tests",	233	"/usr/bin/env", "ruby", "Tools/Scripts/run-jsc-stress-tests",
Lines 218-224 if ($runJSCStress) { Tools/Scripts/run-javascriptcore-tests_sec4
218	"LayoutTests/fast/js/regress/script-tests"	237	"LayoutTests/fast/js/regress/script-tests"
219	);	238	);
220	print "Running: " . join(" ", @jscStressDriverCmd) . "\n";	239	print "Running: " . join(" ", @jscStressDriverCmd) . "\n";
221	$result = system(@jscStressDriverCmd);	240	my $result = system(@jscStressDriverCmd);
222	exit exitStatus($result) if $result;	241	exit exitStatus($result) if $result;
223	}	242	}
224		243
Lines 296-307 if ($numJSCStressFailures) { Tools/Scripts/run-javascriptcore-tests_sec5
296		315
297	print "\n";	316	print "\n";
298		317
299	print "Results for Mozilla tests:\n";	318	if ($runMozilla) {
300	printThingsFound($numNewMozillaFailures, "regression", "regressions", "found");	319	print "Results for Mozilla tests:\n";
301	printThingsFound($numOldMozillaFailures, "test", "tests", "fixed");	320	printThingsFound($numNewMozillaFailures, "regression", "regressions", "found");
302	print " OK.\n" if $numNewMozillaFailures == 0;	321	printThingsFound($numOldMozillaFailures, "test", "tests", "fixed");
303		322	print " OK.\n" if $numNewMozillaFailures == 0;
304	print "\n";	323
		324	print "\n";
		325	}
305		326
306	if ($runFastJS) {	327	if ($runFastJS) {
307	print "Results for fast/js tests:\n";	328	print "Results for fast/js tests:\n";
Lines 320-323 if ($runJSCStress) { Tools/Scripts/run-javascriptcore-tests_sec6
320	print "\n";	341	print "\n";
321	}	342	}
322		343
323	exit(1) if $numNewMozillaFailures \|\| $numJSFailures \|\| $numJSCrashes \|\| $numJSCStressFailures;	344	exit(1) if ($runMozilla && $numNewMozillaFailures) \|\| ($runFastJS && ($numJSFailures \|\| $numJSCrashes)) \|\| ($runJSCStress && $numJSCStressFailures);

Lines 1-3 Tools/ChangeLog_sec1
- Tools/ChangeLog +9 lines
		1	2013-09-06 Filip Pizlo <fpizlo@apple.com>
		2
		3	It should be possible to run-javascriptcore-tests with only a subset of the test suites
		4	https://bugs.webkit.org/show_bug.cgi?id=120918
		5
		6	Reviewed by NOBODY (OOPS!).
		7
		8	* Scripts/run-javascriptcore-tests:
		9
1	2013-09-06 Filip Pizlo <fpizlo@apple.com>	10	2013-09-06 Filip Pizlo <fpizlo@apple.com>
2		11
3	The 'failed' file should report collectionName/testName rather than just	12	The 'failed' file should report collectionName/testName rather than just