LIVE / thrust /internal /test /thrust_nightly.pl
Xu Ma
update
1c3c0d9
raw
history blame
19.9 kB
#! /usr/bin/perl
###############################################################################
# Copyright (c) 2018 NVIDIA Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
###############################################################################
use strict;
use warnings;
print(`perl --version`);
use Getopt::Long;
use Cwd;
use Cwd "abs_path";
use Config; # For signal names and numbers.
use IPC::Open2;
use File::Temp;
use POSIX "strftime";
my $have_time_hi_res = 0;
if (eval { require Time::HiRes })
{
printf("#### CONFIG timestamp `gettimeofday`\n");
import Time::HiRes "gettimeofday";
$have_time_hi_res = 1;
} else {
printf("#### CONFIG timestamp `time`\n");
}
sub timestamp()
{
if ($have_time_hi_res) {
return gettimeofday();
} else {
return time();
}
}
my %CmdLineOption;
my $arch = "";
my $abi = "";
my $os = "";
my $build = "release";
my $bin_path;
my $filecheck_path;
my $filecheck_data_path = "internal/test";
my $timeout_min = 15;
# https://stackoverflow.com/questions/29862178/name-of-signal-number-2
my @sig_names;
@sig_names[ split ' ', $Config{sig_num} ] = split ' ', $Config{sig_name};
my %sig_nums;
@sig_nums{ split ' ', $Config{sig_name} } = split ' ', $Config{sig_num};
if (`uname` =~ m/CYGWIN/) {
$os = "win32";
} elsif ($^O eq "MSWin32") {
$os = "win32";
} else {
$os = `uname`;
chomp($os);
}
if ($os eq "win32") {
$ENV{'PROCESSOR_ARCHITECTURE'} ||= "";
$ENV{'PROCESSOR_ARCHITEW6432'} ||= "";
if ((lc($ENV{PROCESSOR_ARCHITECTURE}) ne "x86") ||
(lc($ENV{PROCESSOR_ARCHITECTURE}) eq "amd64") ||
(lc($ENV{PROCESSOR_ARCHITEW6432}) eq "amd64")) {
$arch = "x86_64";
} else {
$arch = "i686";
}
} else {
$arch = `uname -m`;
chomp($arch);
}
sub usage()
{
printf("Usage: thrust_nightly.pl <options>\n");
printf("Options:\n");
printf(" -help : Print help message\n");
printf(" -forcearch <arch> : i686|x86_64|ARMv7|aarch64 (default: $arch)\n");
printf(" -forceabi <abi> : Specify abi to be used for arm (gnueabi|gnueabihf)\n");
printf(" -forceos <os> : win32|Linux|Darwin (default: $os)\n");
printf(" -build <release|debug> : (default: debug)\n");
printf(" -bin-path <path> : Specify location of test binaries\n");
printf(" -filecheck-path <path> : Specify location of filecheck binary\n");
printf(" -filecheck-data-path <path> : Specify location of filecheck data (default: $filecheck_data_path)\n");
printf(" -timeout-min <min> : timeout in minutes for each individual test\n");
}
GetOptions(\%CmdLineOption,
'help' => sub { usage() and exit 0 },
"forcearch=s" => \$arch,
"forceabi=s" => \$abi,
"forceos=s" => \$os,
"build=s" => \$build,
"bin-path=s" => \$bin_path,
"filecheck-path=s" => \$filecheck_path,
"filecheck-data-path=s" => \$filecheck_data_path,
"timeout-min=i" => \$timeout_min,
);
my $pwd = getcwd();
my $bin_path_root = abs_path ("${pwd}/..");
if ($arch eq "ARMv7") {
if ($abi eq "") {
$abi = "_gnueabi"; #Use default abi for arm if not specified
}
else {
$abi = "_${abi}";
}
}
else {
$abi = ""; #Ignore abi for architectures other than arm
}
my $uname = "";
$uname = $arch;
chomp($uname);
if (not $bin_path) {
$bin_path = "${bin_path_root}/bin/${uname}_${os}${abi}_${build}";
}
if (not $filecheck_path) {
$filecheck_path = "${bin_path}/nvvm/tools";
}
sub process_return_code {
my ($name, $ret, $msg) = @_;
if ($ret != 0) {
my $signal = $ret & 127;
my $app_exit = $ret >> 8;
my $dumped_core = $ret & 0x80;
if (($app_exit != 0) && ($app_exit != 0)) {
if ($msg ne "") {
printf("#### ERROR $name exited with return value $app_exit. $msg\n");
} else {
printf("#### ERROR $name exited with return value $app_exit.\n");
}
}
if ($signal != 0) {
if ($msg ne "") {
printf("#### ERROR $name received signal SIG$sig_names[$signal] ($signal). $msg\n");
} else {
printf("#### ERROR $name received signal SIG$sig_names[$signal] ($signal).\n");
}
if ($sig_nums{'INT'} eq $signal) {
die("Terminating testing due to SIGINT.");
}
}
if ($dumped_core != 0) {
if ($msg ne "") {
printf("#### ERROR $name generated a core dump. $msg\n");
} else {
printf("#### ERROR $name generated a core dump.\n");
}
}
}
}
my $have_filecheck = 1;
sub filecheck_sanity {
my $filecheck_cmd = "$filecheck_path/FileCheck $filecheck_data_path/thrust.sanity.filecheck";
my $filecheck_pid = open(my $filecheck_stdin, "|-", "$filecheck_cmd 2>&1");
print $filecheck_stdin "SANITY";
my $filecheck_ret = 0;
if (close($filecheck_stdin) == 0)
{
$filecheck_ret = $?;
}
if ($filecheck_ret == 0) {
printf("#### SANE FileCheck\n");
} else {
# Use a temporary file to send the output to
# FileCheck so we can get the output this time,
# because Perl and bidirectional pipes suck.
my $tmp = File::Temp->new();
my $tmp_filename = $tmp->filename;
print $tmp "SANITY";
printf("********************************************************************************\n");
print `$filecheck_cmd -input-file $tmp_filename`;
printf("********************************************************************************\n");
process_return_code("FileCheck Sanity", $filecheck_ret, "");
printf("#### INSANE FileCheck\n");
$have_filecheck = 0;
}
}
# Wrapper for system that logs the commands so you can see what it did
sub run_cmd {
my ($cmd) = @_;
my $ret = 0;
my @executable;
my @output;
my $syst_cmd;
my $start = timestamp();
eval {
local $SIG{ALRM} = sub { die("Command timed out (received SIGALRM).\n") };
alarm (60 * $timeout_min);
$syst_cmd = $cmd;
@executable = split(' ', $syst_cmd, 2);
open(my $child, "-|", "$syst_cmd") or die("Could not execute $syst_cmd.\n");
if ($child)
{
@output = <$child>;
}
if (close($child) == 0)
{
$ret = $?;
}
alarm 0;
};
my $elapsed = timestamp() - $start;
if ($@) {
printf("\n#### ERROR Command timeout reached, killing $executable[0].\n");
system("killall ".$executable[0]);
return ($sig_nums{'KILL'}, $elapsed, @output);
}
return ($ret, $elapsed, @output);
}
sub current_time
{
return strftime("%x %X %Z", localtime());
}
my $failures = 0;
my $known_failures = 0;
my $errors = 0;
my $passes = 0;
sub run_examples {
# Get list of tests in binary folder.
my $dir = cwd();
chdir $bin_path;
my @examplelist;
if ($os eq "win32")
{
@examplelist = glob('thrust.example.*.exe');
} else {
@examplelist = glob('thrust.example.*');
}
chdir $dir;
my $test;
foreach $test (@examplelist)
{
my $test_exe = $test;
# Ignore FileCheck files.
if ($test =~ /[.]filecheck$/)
{
next;
}
if ($os eq "win32")
{
$test =~ s/\.exe//g;
}
# Check the test actually exists.
if (!-e "${bin_path}/${test_exe}")
{
next;
}
my $cmd = "${bin_path}/${test_exe} --verbose 2>&1";
printf("&&&& RUNNING $test\n");
printf("#### CURRENT_TIME " . current_time() . "\n");
my ($ret, $elapsed, @output) = run_cmd($cmd);
printf("********************************************************************************\n");
print @output;
printf("********************************************************************************\n");
if ($ret != 0) {
process_return_code($test, $ret, "Example crash?");
printf("&&&& FAILED $test\n");
printf("#### WALLTIME $test %.2f [s]\n", $elapsed);
$errors = $errors + 1;
} else {
printf("&&&& PASSED $test\n");
printf("#### WALLTIME $test %.2f [s]\n", $elapsed);
$passes = $passes + 1;
if ($have_filecheck) {
# Check output with LLVM FileCheck.
printf("&&&& RUNNING FileCheck $test\n");
if (-f "${filecheck_data_path}/${test}.filecheck") {
# If the filecheck file is empty, don't use filecheck, just
# check if the output file is also empty.
if (-z "${filecheck_data_path}/${test}.filecheck") {
if (join("", @output) eq "") {
printf("&&&& PASSED FileCheck $test\n");
$passes = $passes + 1;
} else {
printf("#### ERROR Output received but not expected.\n");
printf("&&&& FAILED FileCheck $test\n");
$failures = $failures + 1;
}
} else {
my $filecheck_cmd = "$filecheck_path/FileCheck $filecheck_data_path/$test.filecheck";
my $filecheck_pid = open(my $filecheck_stdin, "|-", "$filecheck_cmd 2>&1");
print $filecheck_stdin @output;
my $filecheck_ret = 0;
if (close($filecheck_stdin) == 0)
{
$filecheck_ret = $?;
}
if ($filecheck_ret == 0) {
printf("&&&& PASSED FileCheck $test\n");
$passes = $passes + 1;
} else {
# Use a temporary file to send the output to
# FileCheck so we can get the output this time,
# because Perl and bidirectional pipes suck.
my $tmp = File::Temp->new();
my $tmp_filename = $tmp->filename;
print $tmp @output;
printf("********************************************************************************\n");
print `$filecheck_cmd -input-file $tmp_filename`;
printf("********************************************************************************\n");
process_return_code("FileCheck $test", $filecheck_ret, "");
printf("&&&& FAILED FileCheck $test\n");
$failures = $failures + 1;
}
}
} else {
printf("#### ERROR $test has no FileCheck comparison.\n");
printf("&&&& FAILED FileCheck $test\n");
$errors = $errors + 1;
}
}
}
printf("\n");
}
}
sub run_unit_tests {
# Get list of tests in binary folder.
my $dir = cwd();
chdir $bin_path;
my @unittestlist;
if ($os eq "win32")
{
@unittestlist = glob('thrust.test.*.exe');
} else {
@unittestlist = glob('thrust.test.*');
}
chdir $dir;
my $test;
foreach $test (@unittestlist)
{
my $test_exe = $test;
# Ignore FileCheck files.
if ($test =~ /[.]filecheck$/)
{
next;
}
if ($os eq "win32")
{
$test =~ s/\.exe//g;
}
# Check the test actually exists.
if (!-e "${bin_path}/${test_exe}")
{
next;
}
# Check the test actually exists
next unless (-e "${bin_path}/${test_exe}");
my $cmd = "${bin_path}/${test_exe} --verbose 2>&1";
printf("&&&& RUNNING $test\n");
printf("#### CURRENT_TIME " . current_time() . "\n");
my ($ret, $elapsed, @output) = run_cmd($cmd);
printf("********************************************************************************\n");
print @output;
printf("********************************************************************************\n");
my $fail = 0;
my $known_fail = 0;
my $error = 0;
my $pass = 0;
my $found_totals = 0;
foreach my $line (@output)
{
if (($fail, $known_fail, $error, $pass) = $line =~ /Totals: ([0-9]+) failures, ([0-9]+) known failures, ([0-9]+) errors, and ([0-9]+) passes[.]/igs) {
$found_totals = 1;
$failures = $failures + $fail;
$known_failures = $known_failures + $known_fail;
$errors = $errors + $error;
$passes = $passes + $pass;
last;
} else {
$fail = 0;
$known_fail = 0;
$error = 0;
$pass = 0;
}
}
if ($ret == 0) {
if ($found_totals == 0) {
$errors = $errors + 1;
printf("#### ERROR $test returned 0 and no summary line was found. Invalid test?\n");
printf("&&&& FAILED $test\n");
printf("#### WALLTIME $test %.2f [s]\n", $elapsed);
}
else {
if ($fail != 0 or $error != 0) {
$errors = $errors + 1;
printf("#### ERROR $test returned 0 and had failures or errors. Test driver error?\n");
printf("&&&& FAILED $test\n");
printf("#### WALLTIME $test %.2f [s]\n", $elapsed);
} elsif ($known_fail == 0 and $pass == 0) {
printf("#### DISABLED $test returned 0 and had no failures, known failures, errors or passes.\n");
printf("&&&& PASSED $test\n");
printf("#### WALLTIME $test %.2f [s]\n", $elapsed);
} else {
printf("&&&& PASSED $test\n");
printf("#### WALLTIME $test %.2f [s]\n", $elapsed);
if ($have_filecheck) {
# Check output with LLVM FileCheck if the test has a FileCheck input.
if (-f "${filecheck_data_path}/${test}.filecheck") {
printf("&&&& RUNNING FileCheck $test\n");
# If the filecheck file is empty, don't use filecheck,
# just check if the output file is also empty.
if (! -z "${filecheck_data_path}/${test}.filecheck") {
if (@output) {
printf("&&&& PASSED FileCheck $test\n");
$passes = $passes + 1;
} else {
printf("#### Output received but not expected.\n");
printf("&&&& FAILED FileCheck $test\n");
$failures = $failures + 1;
}
} else {
my $filecheck_cmd = "$filecheck_path/FileCheck $filecheck_data_path/$test.filecheck";
my $filecheck_pid = open(my $filecheck_stdin, "|-", "$filecheck_cmd 2>&1");
print $filecheck_stdin @output;
my $filecheck_ret = 0;
if (close($filecheck_stdin) == 0)
{
$filecheck_ret = $?;
}
if ($filecheck_ret == 0) {
printf("&&&& PASSED FileCheck $test\n");
$passes = $passes + 1;
} else {
# Use a temporary file to send the output to
# FileCheck so we can get the output this time,
# because Perl and bidirectional pipes suck.
my $tmp = File::Temp->new();
my $tmp_filename = $tmp->filename;
print $tmp @output;
printf("********************************************************************************\n");
print `$filecheck_cmd -input-file $tmp_filename`;
printf("********************************************************************************\n");
process_return_code("FileCheck $test", $filecheck_ret, "");
printf("&&&& FAILED FileCheck $test\n");
$failures = $failures + 1;
}
}
}
}
}
}
} else {
$errors = $errors + 1;
process_return_code($test, $ret, "Test crash?");
printf("&&&& FAILED $test\n");
printf("#### WALLTIME $test %.2f [s]\n", $elapsed);
}
printf("\n");
}
}
sub dvs_summary {
my $dvs_score = 0;
my $denominator = $failures + $known_failures + $errors + $passes;
if ($denominator == 0) {
$dvs_score = 0;
}
else {
$dvs_score = 100 * (($passes + $known_failures) / $denominator);
}
printf("\n");
printf("%*%*%*%* FA!LUR3S $failures\n");
printf("%*%*%*%* KN0WN FA!LUR3S $known_failures\n");
printf("%*%*%*%* 3RR0RS $errors\n");
printf("%*%*%*%* PASS3S $passes\n");
printf("\n");
printf("CUDA DVS BASIC SANITY SCORE : %.1f\n", $dvs_score);
if ($failures + $errors > 0) {
exit(1);
}
}
###############################################################################
printf("#### CONFIG arch `%s`\n", $arch);
printf("#### CONFIG abi `%s`\n", $abi);
printf("#### CONFIG os `%s`\n", $os);
printf("#### CONFIG build `%s`\n", $build);
printf("#### CONFIG bin_path `%s`\n", $bin_path);
printf("#### CONFIG have_filecheck `$have_filecheck`\n");
printf("#### CONFIG filecheck_path `%s`\n", $filecheck_path);
printf("#### CONFIG filecheck_data_path `%s`\n", $filecheck_data_path);
printf("#### CONFIG have_time_hi_res `$have_time_hi_res`\n");
printf("#### CONFIG timeout_min `%s`\n", $timeout_min);
printf("#### ENV PATH `%s`\n", defined $ENV{'PATH'} ? $ENV{'PATH'} : '');
printf("#### ENV LD_LIBRARY_PATH `%s`\n", defined $ENV{'LD_LIBRARY_PATH'} ? $ENV{'LD_LIBRARY_PATH'} : '');
printf("\n");
filecheck_sanity();
printf("\n");
my $START_TIME = current_time();
run_examples();
run_unit_tests();
my $STOP_TIME = current_time();
printf("#### START_TIME $START_TIME\n");
printf("#### STOP_TIME $STOP_TIME\n");
dvs_summary();