X-Git-Url: https://pintos-os.org/cgi-bin/gitweb.cgi?a=blobdiff_plain;f=grading%2Fuserprog%2Frun-tests;h=4f4bb100a55ae74d2788242bef6d753e9addb7a4;hb=06aa2c0d714c83f0300559b48972bf454f3dab56;hp=c51089eb023234a0e95910851995b5d8100cb0fb;hpb=2e2f580dd9eb1a8fa094e35f4b967e2547774d9c;p=pintos-anon diff --git a/grading/userprog/run-tests b/grading/userprog/run-tests index c51089e..4f4bb10 100755 --- a/grading/userprog/run-tests +++ b/grading/userprog/run-tests @@ -51,6 +51,8 @@ sub usage { write-normal write-bad-ptr write-boundary write-zero write-stdin write-bad-fd exec-once exec-arg exec-multiple exec-missing exec-bad-ptr + join-simple join-twice join-killed join-bad-pid + multi-recurse multi-oom multi-child-fd ) unless @TESTS > 0; our (%args); @@ -59,6 +61,8 @@ for my $key ('args-argc', 'args-argv0', 'args-argvn', 'args-multiple') { } $args{'args-single'} = "onearg"; $args{'args-dbl-space'} = "two args"; +$args{'multi-recurse'} = "15"; +$args{'multi-oom'} = "0"; # Handle final grade mode. if ($grade) { @@ -185,10 +189,24 @@ sub extract_tarball { xsystem ("cd pintos/src && tar xzf ../../$tarball", DIE => "extraction failed\n"); + if (-e "fixme.sh") { + print "Running fixme.sh...\n"; + xsystem ("sh -e fixme.sh", DIE => "fix script failed\n"); + } + print "Patching...\n"; xsystem ("patch -fs pintos/src/lib/debug.c < $GRADES_DIR/panic.diff", LOG => "patch", DIE => "patch failed\n"); + xsystem ("patch -fs pintos/src/lib/kernel/bitmap.c " + . "< $GRADES_DIR/random.diff", + LOG => "patch", + DIE => "patch failed\n"); + + open (CONSTANTS, ">pintos/src/constants.h") + or die "constants.h: create: $!\n"; + print CONSTANTS "#define THREAD_JOIN_IMPLEMENTED 1\n"; + close CONSTANTS; } sub ext_mdyHMS { @@ -257,6 +275,7 @@ sub really_run_test { # Run. my ($timeout) = 10; + $timeout = 600 if $test =~ /^multi-/; my ($testargs) = defined ($args{$test}) ? " $args{$test}" : ""; xsystem ("pintos " . "--os-disk=pintos/src/userprog/build/os.dsk " @@ -322,207 +341,55 @@ sub grade_write_normal { } } -sub get_file { - my ($guest_fn, $host_fn) = @_; - xsystem ("pintos " - . "--os-disk=pintos/src/userprog/build/os.dsk " - . "--fs-disk=output/$test/fs.dsk " - . "-v get $guest_fn $host_fn", - LOG => "$test/get-$guest_fn", - TIMEOUT => 10) - or die "get $guest_fn failed\n"; -} - -sub grade_alarm_negative { - my (@output) = @_; - verify_common (@output); - die "Crashed in timer_sleep()\n" if !grep (/^Success\.$/, @output); -} - -sub grade_join_invalid { +sub grade_multi_oom { my (@output) = @_; verify_common (@output); - grep (/Testing invalid join/, @output) or die "Test didn't start\n"; - grep (/Invalid join test done/, @output) or die "Test didn't complete\n"; -} -sub grade_join_no { - my (@output) = @_; - verify_common (@output); - grep (/Testing no join/, @output) or die "Test didn't start\n"; - grep (/No join test done/, @output) or die "Test didn't complete\n"; -} - -sub grade_join_multiple { - my (@output) = @_; - - verify_common (@output); - my (@t); - $t[4] = $t[5] = $t[6] = -1; - local ($_); - foreach (@output) { - my ($idx) = /^Thread (\d+)/ or next; - my ($iter) = /iteration (\d+)$/; - $iter = 5 if /done!$/; - die "Malformed output\n" if !defined $iter; - if ($idx == 6) { - die "Thread 6 started before either other thread finished\n" - if $t[4] < 5 && $t[5] < 5; - die "Thread 6 started before thread 4 finished\n" - if $t[4] < 5; - die "Thread 6 started before thread 5 finished\n" - if $t[5] < 5; - } - die "Thread $idx out of order output\n" if $t[$idx] != $iter - 1; - $t[$idx] = $iter; + @output = fix_exit_codes (get_core_output (@output)); + my ($n) = 0; + while (my ($m) = $output[0] =~ /^\(multi-oom\) begin (\d+)$/) { + die "Child process $m started out of order.\n" if $m != $n; + $n = $m + 1; + shift @output; } - - my ($err) = ""; - for my $idx (4, 5, 6) { - if ($t[$idx] == -1) { - $err .= "Thread $idx did not run at all\n"; - } elsif ($t[$idx] != 5) { - $err .= "Thread $idx only completed $t[$idx] iterations\n"; - } + die "Only $n child process(es) started.\n" if $n < 15; + + # There could be a death notice for a process that didn't get + # fully loaded, and/or notices from the loader. + while (@output > 0 + && ($output[0] =~ /^multi-oom: exit\(-1\)$/ + || $output[0] =~ /^load: /)) { + shift @output; } - die $err if $err ne ''; -} -sub grade_priority_fifo { - my (@output) = @_; + while (--$n >= 0) { + die "Output ended unexpectedly before process $n finished.\n" + if @output < 2; - verify_common (@output); - my ($thread_cnt) = 10; - my ($iter_cnt) = 5; - my (@order); - my (@t) = (-1) x $thread_cnt; - local ($_); - foreach (@output) { - my ($idx) = /^Thread (\d+)/ or next; - my ($iter) = /iteration (\d+)$/; - $iter = $iter_cnt if /done!$/; - die "Malformed output\n" if !defined $iter; - if (@order < $thread_cnt) { - push (@order, $idx); - die "Thread $idx repeated within first $thread_cnt iterations: " - . join (' ', @order) . ".\n" - if grep ($_ == $idx, @order) != 1; - } else { - die "Thread $idx ran when $order[0] should have.\n" - if $idx != $order[0]; - push (@order, shift @order); - } - die "Thread $idx out of order output.\n" if $t[$idx] != $iter - 1; - $t[$idx] = $iter; - } + local ($_); + chomp ($_ = shift @output); + die "Found '$_' expecting 'end' message.\n" if !/^\(multi-oom\) end/; + die "Child process $n ended out of order.\n" + if !/^\(multi-oom\) end $n$/; - my ($err) = ""; - for my $idx (0..$#t) { - if ($t[$idx] == -1) { - $err .= "Thread $idx did not run at all.\n"; - } elsif ($t[$idx] != $iter_cnt) { - $err .= "Thread $idx only completed $t[$idx] iterations.\n"; - } + chomp ($_ = shift @output); + die "Kernel didn't print proper exit message for process $n.\n" + if !/^multi-oom: exit\($n\)$/; } - die $err if $err ne ''; -} - -sub grade_mlfqs_on { - my (@output) = @_; - verify_common (@output); - our (@mlfqs_on_stats) = mlfqs_stats (@output); -} - -sub grade_mlfqs_off { - my (@output) = @_; - verify_common (@output); - our (@mlfqs_off_stats) = mlfqs_stats (@output); + die "Spurious output at end: '$output[0]'.\n" if @output; } -sub grade_mlfqs_speedup { - our (@mlfqs_off_stats); - our (@mlfqs_on_stats); - eval { - check_mlfqs (); - my ($off_ticks) = $mlfqs_off_stats[1]; - my ($on_ticks) = $mlfqs_on_stats[1]; - die "$off_ticks ticks without MLFQS, $on_ticks with MLFQS\n" - if $on_ticks >= $off_ticks; - die "ok\n"; - }; - chomp $@; - $result{'mlfqs-speedup'} = $@; -} - -sub grade_mlfqs_priority { - our (@mlfqs_off_stats); - our (@mlfqs_on_stats); - eval { - check_mlfqs () if !defined (@mlfqs_on_stats); - for my $cat qw (CPU IO MIX) { - die "Priority changed away from PRI_DEFAULT (29) without MLFQS\n" - if $mlfqs_off_stats[0]{$cat}{MIN} != 29 - || $mlfqs_off_stats[0]{$cat}{MAX} != 29; - die "Minimum priority never changed from PRI_DEFAULT (29) " - . "with MLFQS\n" - if $mlfqs_on_stats[0]{$cat}{MIN} == 29; - die "Maximum priority never changed from PRI_DEFAULT (29) " - . "with MLFQS\n" - if $mlfqs_on_stats[0]{$cat}{MAX} == 29; - } - die "ok\n"; - }; - chomp $@; - $result{'mlfqs-priority'} = $@; -} - -sub check_mlfqs { - our (@mlfqs_off_stats); - our (@mlfqs_on_stats); - die "p1-4 didn't finish with MLFQS on or off\n" - if !defined (@mlfqs_off_stats) && !defined (@mlfqs_on_stats); - die "p1-4 didn't finish with MLFQS on\n" - if !defined (@mlfqs_on_stats); - die "p1-4 didn't finish with MLFQS off\n" - if !defined (@mlfqs_off_stats); +sub get_file { + my ($guest_fn, $host_fn) = @_; + xsystem ("pintos " + . "--os-disk=pintos/src/userprog/build/os.dsk " + . "--fs-disk=output/$test/fs.dsk " + . "-v get $guest_fn $host_fn", + LOG => "$test/get-$guest_fn", + TIMEOUT => 10) + or die "get $guest_fn failed\n"; } -sub mlfqs_stats { - my (@output) = @_; - my (%stats) = (CPU => {}, IO => {}, MIX => {}); - my (%map) = ("CPU intensive" => 'CPU', - "IO intensive" => 'IO', - "Alternating IO/CPU" => 'MIX'); - my (%rmap) = reverse %map; - my ($ticks); - local ($_); - foreach (@output) { - $ticks = $1 if /Timer: (\d+) ticks/; - my ($thread, $pri) = /^([A-Za-z\/ ]+): (\d+)$/ or next; - my ($t) = $map{$thread} or next; - - my ($s) = $stats{$t}; - $$s{N}++; - $$s{SUM} += $pri; - $$s{SUM2} += $pri * $pri; - $$s{MIN} = $pri if !defined ($$s{MIN}) || $pri < $$s{MIN}; - $$s{MAX} = $pri if !defined ($$s{MAX}) || $pri > $$s{MAX}; - } - - my (%expect_n) = (CPU => 5000, IO => 1000, MIX => 12000); - for my $cat (values (%map)) { - my ($s) = $stats{$cat}; - die "$rmap{$cat} printed $$s{N} times, not $expect_n{$cat}\n" - if $$s{N} != $expect_n{$cat}; - die "$rmap{$cat} priority dropped to $$s{MIN}, below PRI_MIN (0)\n" - if $$s{MIN} < 0; - die "$rmap{$cat} priority rose to $$s{MAX}, above PRI_MAX (59)\n" - if $$s{MAX} > 59; - $$s{MEAN} = $$s{SUM} / $$s{N}; - } - - return (\%stats, $ticks); -} sub verify_common { my (@output) = @_; @@ -544,7 +411,7 @@ sub verify_common { } else { $A2L = "i386-elf-addr2line"; } - open (A2L, "$A2L -fe output/$test/kernel.o @addrs|"); + open (A2L, "$A2L -fe pintos/src/userprog/build/kernel.o @addrs|"); for (;;) { my ($function, $line); last unless defined ($function = ); @@ -558,6 +425,25 @@ sub verify_common { die "Kernel panic. Details at end of file.\n" } + if (grep (/Pintos booting/, @output) > 1) { + my ($details); + + $details = "Pintos spontaneously rebooted during this test.\n"; + $details .= "This is most often due to unhandled page faults.\n"; + $details .= "Here's the output from the initial boot through the\n"; + $details .= "first reboot:\n\n"; + + my ($i) = 0; + local ($_); + for (@output) { + $details .= " $_\n"; + last if /Pintos booting/ && ++$i > 1; + } + $details{$test} = $details; + die "Triple-fault caused spontaneous reboot(s). " + . "Details at end of file.\n"; + } + die "No output at all\n" if @output == 0; die "Didn't start up properly: no \"Pintos booting\" startup message\n" if !grep (/Pintos booting with.*kB RAM\.\.\./, @output); @@ -593,20 +479,36 @@ sub get_core_output { return @output[$first ... $last]; } -sub compare_output { - my ($exp, @actual) = @_; - @actual = get_core_output (map ("$_\n", @actual)); +sub fix_exit_codes { + my (@output) = @_; # Fix up lines that look like exit codes. - for my $i (0...$#actual) { - if (my ($process, $code) - = $actual[$i] =~ /^([-a-zA-Z0-9 ]+):.*[ \(](-?\d+)\b\)?$/) { + # Exit codes are supposed to be printed in the form "process: exit(code)" + # but people get unfortunately creative with it. + for my $i (0...$#output) { + local ($_) = $output[$i]; + + my ($process, $code); + if ((($process, $code) = /^([-a-z0-9 ]+):.*[ \(](-?\d+)\b\)?$/) + || (($process, $code) = /^([-a-z0-9 ]+) exit\((-?\d+)\)$/) + || (($process, $code) + = /^([-a-z0-9 ]+) \(.*\): exit\((-?\d+)\)$/) + || (($process, $code) = /^([-a-z0-9 ]+):\( (-?\d+) \) $/) + || (($code, $process) = /^shell: exit\((-?\d+)\) \| ([-a-z0-9]+)/) +) { $process = substr ($process, 0, 15); $process =~ s/\s.*//; - $actual[$i] = "$process: exit($code)\n"; + $output[$i] = "$process: exit($code)\n"; } } + return @output; +} + +sub compare_output { + my ($exp, @actual) = @_; + @actual = fix_exit_codes (get_core_output (map ("$_\n", @actual))); + my ($details) = ""; $details .= "$test actual output:\n"; $details .= join ('', map (" $_", @actual)); @@ -655,7 +557,22 @@ sub compare_output { $details .= "Differences in `diff -u' format:\n"; $details .= join ('', @diff); - $details .= "(This is considered a `fuzzy match'.)\n" if $fuzzy_match; + $details .= "(This is considered a `fuzzy match'.)\n" + if !$not_fuzzy_match; + } + + if ($fuzzy_match) { + $details = + "This test passed, but with extra, unexpected output.\n" + . "Please inspect your code to make sure that it does not\n" + . "produce output other than as specified in the project\n" + . "description.\n\n" + . "$details"; + } else { + $details = + "This test failed because its output did not match any\n" + . "of the acceptable form(s).\n\n" + . "$details"; } $details{$test} = $details; @@ -669,14 +586,26 @@ sub write_grades { my ($ploss) = 0; my ($tloss) = 0; my ($total) = 0; + my ($warnings) = 0; for (my ($i) = 0; $i <= $#summary; $i++) { local ($_) = $summary[$i]; if (my ($loss, $test) = /^ -(\d+) ([-a-zA-Z0-9]+):/) { my ($result) = $result{$test} || "Not tested."; if ($result eq 'ok') { - splice (@summary, $i, 1); - $i--; + if (!defined $details{$test}) { + # Test successful and no warnings. + splice (@summary, $i, 1); + $i--; + } else { + # Test successful with warnings. + s/-(\d+) //; + $summary[$i] = $_; + splice (@summary, $i + 1, 0, + " Test passed with warnings. " + . "Details at end of file."); + $warnings++; + } } else { $ploss += $loss; $tloss += $loss; @@ -686,8 +615,10 @@ sub write_grades { } elsif (my ($ptotal) = /^Score: \/(\d+)$/) { $total += $ptotal; $summary[$i] = "Score: " . ($ptotal - $ploss) . "/$ptotal"; - splice (@summary, $i, 0, " All tests passed.") if $ploss == 0; + splice (@summary, $i, 0, " All tests passed.") + if $ploss == 0 && !$warnings; $ploss = 0; + $warnings = 0; $i++; } } @@ -708,8 +639,15 @@ sub write_details { my ($details) = $details{$test}; next if !defined ($details) && ! -e "output/$test/run.out"; + my ($banner); + if ($result{$test} ne 'ok') { + $banner = "$test failure details"; + } else { + $banner = "$test warnings"; + } + print DETAILS "\n" if $n++; - print DETAILS "--- $test details ", '-' x (50 - length ($test)); + print DETAILS "--- $banner ", '-' x (50 - length ($banner)); print DETAILS "\n\n"; if (!defined $details) {