our ($GRADES_DIR);
our ($verbose);
-our (%args);
our %result;
our %details;
our %extra;
our @TESTS;
-our $clean;
-our $grade;
+our $action;
our $hw;
use POSIX;
-use Getopt::Long;
+use Getopt::Long qw(:config no_ignore_case);
use Algorithm::Diff;
\f
+# We execute lots of subprocesses.
+# Without this, our stdout output can get flushed multiple times,
+# which is harmless but looks bizarre.
+$| = 1;
+
sub parse_cmd_line {
+ my ($do_regex, $no_regex);
GetOptions ("v|verbose+" => \$verbose,
"h|help" => sub { usage (0) },
- "t|test=s" => \@TESTS,
- "c|clean" => \$clean,
- "g|grade" => \$grade)
+ "d|do-tests=s" => \$do_regex,
+ "n|no-tests=s" => \$no_regex,
+ "c|clean" => sub { set_action ('clean'); },
+ "x|extract" => sub { set_action ('extract'); },
+ "b|build" => sub { set_action ('build'); },
+ "t|test" => sub { set_action ('test'); },
+ "a|assemble" => sub { set_action ('assemble'); })
or die "Malformed command line; use --help for help.\n";
- die "Non-option argument not supported; use --help for help.\n"
+ die "Non-option arguments not supported; use --help for help.\n"
if @ARGV > 0;
+ @TESTS = split(/,/, join (',', @TESTS)) if defined @TESTS;
+
+ if (!defined $action) {
+ $action = -e 'review.txt' ? 'assemble' : 'test';
+ }
+
+ my (@default_tests) = @_;
+ @TESTS = @default_tests;
+ @TESTS = grep (/$do_regex/, @TESTS) if defined $do_regex;
+ @TESTS = grep (!/$no_regex/, @TESTS) if defined $no_regex;
+}
+
+sub set_action {
+ my ($new_action) = @_;
+ die "actions `$action' and `$new_action' conflict\n"
+ if defined ($action) && $action ne $new_action;
+ $action = $new_action;
}
sub usage {
my ($exitcode) = @_;
- print "run-tests, for grading Pintos projects.\n\n";
- print "Invoke from a directory containing a student tarball named by\n";
- print "the submit script, e.g. username.MMM.DD.YY.hh.mm.ss.tar.gz.\n";
- print "In normal usage, no options are needed.\n\n";
- print "Output is produced in tests.out and details.out.\n\n";
- print "Options:\n";
- print " -c, --clean Remove old output files before starting\n";
- print " -t, --test=TEST Execute TEST only (allowed multiple times)\n";
- print " -g, --grade Instead of running tests, compose grade.out\n";
- print " -v, --verbose Print commands before executing them\n";
- print " -h, --help Print this help message\n";
+ print <<EOF;
+run-tests, for grading Pintos $hw projects.
+
+Invoke from a directory containing a student tarball named by
+the submit script, e.g. username.MMM.DD.YY.hh.mm.ss.tar.gz.
+
+Workflow:
+
+1. Extracts the source tree into pintos/src and applies patches.
+
+2. Builds the source tree. (The threads project modifies and rebuilds
+ the source tree for every test.)
+
+3. Runs the tests on the source tree and grades them. Writes
+ "tests.out" with a summary of the test results, and "details.out"
+ with test failure and warning details.
+
+4. By hand, copy "review.txt" from the tests directory and use it as a
+ template for grading design documents.
+
+5. Assembles "grade.txt", "tests.out", "review.txt", and "tests.out"
+ into "grade.out". This is primarily simple concatenation, but
+ point totals are tallied up as well.
+
+Options:
+ -c, --clean Delete test results and temporary files, then exit.
+ -d, --do-tests=RE Run only tests that match the given regular expression.
+ -n, --no-tests=RE Do not run tests that match the given regular expression.
+ -x, --extract Stop after step 1.
+ -b, --build Stop after step 2.
+ -t, --test Stop after step 3 (default if "review.txt" not present).
+ -a, --assemble Stop after step 5 (default if "review.txt" exists).
+ -v, --verbose Print command lines of subcommands before executing them.
+ -h, --help Print this help message.
+EOF
exit $exitcode;
}
\f
# Source tarballs.
-# Extracts the group's source files into pintos/src,
-# applies any patches providing in the grading directory,
-# and installs a default pintos/src/constants.h
-sub obtain_sources {
- # Nothing to do if we already have a source tree.
+# Extracts the group's source files into pintos/src
+# and applies any patches providing in the grading directory.
+sub extract_sources {
+ # Make sure the output dir exists.
+ -d ("output") || mkdir ("output") or die "output: mkdir: $!\n";
+
+ # Nothing else to do if we already have a source tree.
return if -d "pintos";
my ($tarball) = choose_tarball ();
# Apply patches from grading directory.
# Patches are applied in lexicographic order, so they should
- # probably be named 00-debug.patch, 01-bitmap.patch, etc.
+ # probably be named 00debug.patch, 01bitmap.patch, etc.
# Filenames in patches should be in the format pintos/src/...
print "Patching...\n";
for my $patch (glob ("$GRADES_DIR/patches/*.patch")) {
xsystem ("patch -fs -p0 < $patch",
LOG => $stem, DIE => "applying patch $stem failed\n");
}
-
- # Install default pintos/src/constants.h.
- open (CONSTANTS, ">pintos/src/constants.h")
- or die "constants.h: create: $!\n";
- print CONSTANTS "#define THREAD_JOIN_IMPLEMENTED 1\n";
- close CONSTANTS;
}
# Returns the name of the tarball to extract.
# Sort tarballs in order by time.
@tarballs = sort { ext_mdyHMS ($a) cmp ext_mdyHMS ($b) } @tarballs;
- print "Multiple tarballs:";
+ print "Multiple tarballs:\n";
print "\t$_ submitted ", ext_mdyHMS ($_), "\n" foreach @tarballs;
print "Choosing $tarballs[$#tarballs]\n";
}
return sprintf "%02d-%02d-%02d %02d:%02d:%02d", $y, $m, $d, $H, $M, $S;
}
\f
-# Compiling.
+# Building.
-sub compile {
+sub build {
print "Compiling...\n";
- xsystem ("cd pintos/src/filesys && make", LOG => "make")
- or return "compile error";
+ xsystem ("cd pintos/src/$hw && make", LOG => "make") eq 'ok'
+ or return "Build error";
}
\f
# Run and grade the tests.
for $test (@TESTS) {
print "$test: ";
my ($result) = get_test_result ();
- if ($result eq 'ok') {
- $result = grade_test ($test);
- } elsif ($result =~ /^Timed out/) {
- $result = "$result - " . grade_test ($test);
- }
chomp ($result);
- print "$result";
- print " - with warnings" if $result eq 'ok' && defined $details{$test};
- print "\n";
+
+ my ($grade) = grade_test ($test);
+ chomp ($grade);
- $result{$test} = $result;
+ my ($msg) = $result eq 'ok' ? $grade : "$result - $grade";
+ $msg .= " - with warnings"
+ if $grade eq 'ok' && defined $details{$test};
+ print "$msg\n";
+
+ $result{$test} = $grade;
}
}
open (STDOUT, ">output/$log.out");
open (STDERR, ">output/$log.err");
}
- exec ($command);
+ chdir $options{CHDIR} or die "$options{CHDIR}: chdir: $!\n"
+ if defined ($options{CHDIR});
+ if (!defined ($options{EXEC})) {
+ exec ($command);
+ } else {
+ exec (@{$options{EXEC}});
+ }
exit (-1);
}
waitpid ($pid, 0);
print "Child terminated with signal $signal\n";
}
- my ($exp_status) = !defined ($options{EXPECT}) ? 0 : $options{EXPECT};
- $result = WIFEXITED ($status) && WEXITSTATUS ($status) == $exp_status
- ? "ok" : "error";
+ $result = $status == 0 ? "ok" : "error";
}
-
if ($result eq 'error' && defined $options{DIE}) {
my ($msg) = $options{DIE};
if (defined ($log)) {
rename "output/$test", "output/$test.old" or die "rename: $!\n"
if -d "output/$test";
+ # Make output directory.
+ mkdir "output/$test";
+
# Run the test.
my ($result) = run_test ($test);
+ # Delete any disks in the output directory because they take up
+ # lots of space.
+ unlink (glob ("output/$test/*.dsk"));
+
# Save the results for later.
open (DONE, ">$cache_file") or die "$cache_file: create: $!\n";
print DONE "$result\n";
return $result;
}
-# Creates an output directory for the test,
-# creates all the files needed
-sub run_test {
- # Make output directory.
- mkdir "output/$test";
-
- my ($fs_size) = $test ne 'grow-too-big' ? 2 : .25;
- xsystem ("pintos make-disk output/$test/fs.dsk $fs_size >/dev/null 2>&1",
- DIE => "failed to create file system disk");
- xsystem ("pintos make-disk output/$test/swap.dsk 2 >/dev/null 2>&1",
- DIE => "failed to create swap disk");
-
- # Format disk, install test.
- my ($pintos_base_cmd) =
- "pintos "
- . "--os-disk=pintos/src/filesys/build/os.dsk "
- . "--fs-disk=output/$test/fs.dsk "
- . "--swap-disk=output/$test/swap.dsk "
- . "-v";
- unlink ("output/$test/fs.dsk", "output/$test/swap.dsk"),
- return "format/put error"
- if xsystem ("$pintos_base_cmd put -f $GRADES_DIR/$test $test",
- LOG => "$test/put", TIMEOUT => 60, EXPECT => 1) ne 'ok';
-
- my (@extra_files);
- push (@extra_files, "child-syn-read") if $test eq 'syn-read';
- push (@extra_files, "child-syn-wrt") if $test eq 'syn-write';
- push (@extra_files, "child-syn-rw") if $test eq 'syn-rw';
- for my $fn (@extra_files) {
- return "format/put error"
- if xsystem ("$pintos_base_cmd put $GRADES_DIR/$fn $fn",
- LOG => "$test/put-$fn", TIMEOUT => 60, EXPECT => 1)
- ne 'ok';
+sub run_pintos {
+ my ($cmd_line, %args) = @_;
+ unshift (@$cmd_line, 'pintos');
+ my ($retval) = xsystem (join (' ', @$cmd_line), %args, EXEC => $cmd_line);
+ return 'ok' if $retval eq 'ok';
+ if ($retval eq 'timeout') {
+ my ($msg) = "Timed out after $args{TIMEOUT} seconds";
+ my ($load_avg) = `uptime` =~ /(load average:.*)$/i;
+ $msg .= " - $load_avg" if defined $load_avg;
+ return $msg;
}
-
- # Run.
- my ($timeout) = 120;
- my ($testargs) = defined ($args{$test}) ? " $args{$test}" : "";
- my ($retval) =
- xsystem ("$pintos_base_cmd run -q -ex \"$test$testargs\"",
- LOG => "$test/run", TIMEOUT => $timeout, EXPECT => 1);
- my ($result);
- if ($retval eq 'ok') {
- $result = "ok";
- } elsif ($retval eq 'timeout') {
- $result = "Timed out after $timeout seconds";
- } elsif ($retval eq 'error') {
- $result = "Bochs error";
- } else {
- die;
- }
- unlink ("output/$test/fs.dsk", "output/$test/swap.dsk");
- return $result;
+ return 'Error running Bochs' if $retval eq 'error';
+ die;
}
# Grade the test.
sub grade_test {
# Read test output.
- my (@output) = snarf ("output/$test/run.out");
+ my ($outfile) = "output/$test/run.out";
+ if (! -e $outfile) {
+ if (-s "output/$test/make.err") {
+ # make failed.
+ $details{$test} = snarf ("output/$test/make.err");
+ return "make failed. Error messages at end of file.";
+ }
+ return "preparation for test failed";
+ }
+ my (@output) = snarf ($outfile);
# If there's a function "grade_$test", use it to evaluate the output.
# If there's a file "$GRADES_DIR/$test.exp", compare its contents
# Do final grade.
# Combines grade.txt, tests.out, review.txt, and details.out,
# producing grade.out.
-sub compose_final_grade {
+sub assemble_final_grade {
open (OUT, ">grade.out") or die "grade.out: create: $!\n";
open (GRADE, "<grade.txt") or die "grade.txt: open: $!\n";
} else {
$A2L = "i386-elf-addr2line";
}
- open (A2L, "$A2L -fe pintos/src/filesys/build/kernel.o @addrs|");
+ my ($kernel_o);
+ if ($hw eq 'threads') {
+ $kernel_o = "output/$test/kernel.o";
+ } else {
+ $kernel_o = "pintos/src/$hw/build/kernel.o";
+ }
+ open (A2L, "$A2L -fe $kernel_o @addrs|");
for (;;) {
my ($function, $line);
last unless defined ($function = <A2L>);
my ($first);
for ($first = 0; $first <= $#output; $first++) {
- $first++, last if $output[$first] =~ /^Executing '$test.*':$/;
+ my ($line) = $output[$first];
+ $first++, last
+ if ($hw ne 'threads' && $line =~ /^Executing '$test.*':$/)
+ || ($hw eq 'threads'
+ && grep (/^Boot complete.$/, @output[0...$first - 1])
+ && $line =~ /^\s*$/);
}
my ($last);
close (FILE);
return $equal;
}
+\f
+sub success {
+ for my $test (@TESTS) {
+ return 1 if !defined ($result{$test}) || $result{$test} ne 'ok';
+ }
+ return 0;
+}
1;
projects / pintos-anon / blobdiff