X-Git-Url: https://pintos-os.org/cgi-bin/gitweb.cgi?a=blobdiff_plain;f=grading%2Fuserprog%2Frun-tests;h=dd7cd218b901d2e0fc889f4fe6916aa2e280f255;hb=cfbade23277061d4ab0d26a2fb350b98657e7302;hp=37816499814230bb9aceba85fc9721b1e3d8acc4;hpb=4d4065366785163248112abe34ac9a1ecd4b08b5;p=pintos-anon

diff --git a/grading/userprog/run-tests b/grading/userprog/run-tests
index 3781649..dd7cd21 100755
--- a/grading/userprog/run-tests
+++ b/grading/userprog/run-tests
@@ -48,6 +48,11 @@ sub usage {
 	     close-normal close-twice close-stdin close-stdout close-bad-fd
 	     read-normal read-bad-ptr read-boundary read-zero read-stdout
 	     read-bad-fd
+	     write-normal write-bad-ptr write-boundary write-zero write-stdin
+	     write-bad-fd
+	     exec-once exec-arg exec-multiple exec-missing exec-bad-ptr
+	     join-simple join-twice join-killed join-bad-pid
+	     multi-recurse multi-oom multi-child-fd
 	     ) unless @TESTS > 0;
 
 our (%args);
@@ -56,6 +61,8 @@ for my $key ('args-argc', 'args-argv0', 'args-argvn', 'args-multiple') {
 }
 $args{'args-single'} = "onearg";
 $args{'args-dbl-space'} = "two  args";
+$args{'multi-recurse'} = "15";
+$args{'multi-oom'} = "0";
 
 # Handle final grade mode.
 if ($grade) {
@@ -182,10 +189,24 @@ sub extract_tarball {
     xsystem ("cd pintos/src && tar xzf ../../$tarball",
 	     DIE => "extraction failed\n");
 
+    if (-e "fixme.sh") {
+	print "Running fixme.sh...\n";
+	xsystem ("sh -e fixme.sh", DIE => "fix script failed\n");
+    }
+
     print "Patching...\n";
     xsystem ("patch -fs pintos/src/lib/debug.c < $GRADES_DIR/panic.diff",
 	     LOG => "patch",
 	     DIE => "patch failed\n");
+    xsystem ("patch -fs pintos/src/lib/kernel/bitmap.c "
+	     . "< $GRADES_DIR/random.diff",
+	     LOG => "patch",
+	     DIE => "patch failed\n");
+
+    open (CONSTANTS, ">pintos/src/constants.h")
+	or die "constants.h: create: $!\n";
+    print CONSTANTS "#define THREAD_JOIN_IMPLEMENTED 1\n";
+    close CONSTANTS;
 }
 
 sub ext_mdyHMS {
@@ -254,6 +275,7 @@ sub really_run_test {
 
     # Run.
     my ($timeout) = 10;
+    $timeout = 600 if $test =~ /^multi-/;
     my ($testargs) = defined ($args{$test}) ? " $args{$test}" : "";
     xsystem ("pintos "
 	     . "--os-disk=pintos/src/userprog/build/os.dsk "
@@ -271,15 +293,15 @@ sub grade_test {
 
     my (@output) = snarf ("output/$test/run.out");
 
-    if (-e "$GRADES_DIR/$test.exp") {
+    my ($grade_func) = "grade_$test";
+    $grade_func =~ s/-/_/g;
+    if (-e "$GRADES_DIR/$test.exp" && !defined (&$grade_func)) {
 	eval {
 	    verify_common (@output);
 	    compare_output ("$GRADES_DIR/$test.exp", @output);
 	}
     } else {
-	my ($grade_func);
-	($grade_func = $test) =~ s/-/_/g;
-	eval "grade_$grade_func (\@output)";
+	eval "$grade_func (\@output)";
     }
     if ($@) {
 	die $@ if $@ =~ /at \S+ line \d+$/;
@@ -288,234 +310,86 @@ sub grade_test {
     return "ok";
 }
 
-sub grade_alarm_multiple {
-    verify_alarm (7, @_);
-}
-
-sub verify_alarm {
-    my ($iterations, @output) = @_;
-
+sub grade_write_normal {
+    my (@output) = @_;
     verify_common (@output);
-
-    my (@products);
-    for (my ($i) = 0; $i < $iterations; $i++) {
-	for (my ($t) = 0; $t < 5; $t++) {
-	    push (@products, ($i + 1) * ($t + 1) * 10);
+    compare_output ("$GRADES_DIR/write-normal.exp", @output);
+    my ($test_txt) = "output/$test/test.txt";
+    get_file ("test.txt", $test_txt) if ! -e $test_txt;
+
+    my (@actual) = snarf ($test_txt);
+    my (@expected) = snarf ("$GRADES_DIR/sample.txt");
+
+    my ($eq);
+    if ($#actual == $#expected) {
+	$eq = 1;
+	for my $i (0...$#actual) {
+	    $eq = 0 if $actual[$i] ne $expected[$i];
 	}
+    } else {
+	$eq = 0;
     }
-    @products = sort {$a <=> $b} @products;
-
-    local ($_);
-    foreach (@output) {
-	die $_ if /Out of order/;
-
-	my ($p) = /product=(\d+)$/;
-	next if !defined $p;
+    if (!$eq) {
+	my ($details);
+	$details = "Expected file content:\n";
+	$details .= join ('', map ("  $_\n", @expected));
+	$details .= "Actual file content:\n";
+	$details .= join ('', map ("  $_\n", @actual));
+	$extra{$test} = $details;
 
-	my ($q) = shift (@products);
-	die "Too many wakeups.\n" if !defined $q;
-	die "Out of order wakeups ($p vs. $q).\n" if $p != $q; # FIXME
+	die "File written didn't have expected content.\n";
     }
-    die scalar (@products) . " fewer wakeups than expected.\n"
-	if @products != 0;
-}
-
-sub grade_alarm_zero {
-    my (@output) = @_;
-    verify_common (@output);
-    die "Crashed in timer_sleep()\n" if !grep (/^Success\.$/, @output);
-}
-
-sub grade_alarm_negative {
-    my (@output) = @_;
-    verify_common (@output);
-    die "Crashed in timer_sleep()\n" if !grep (/^Success\.$/, @output);
 }
 
-sub grade_join_invalid {
+sub grade_multi_oom {
     my (@output) = @_;
     verify_common (@output);
-    grep (/Testing invalid join/, @output) or die "Test didn't start\n";
-    grep (/Invalid join test done/, @output) or die "Test didn't complete\n";
-}
-
-sub grade_join_no {
-    my (@output) = @_;
-    verify_common (@output);
-    grep (/Testing no join/, @output) or die "Test didn't start\n";
-    grep (/No join test done/, @output) or die "Test didn't complete\n";
-}
 
-sub grade_join_multiple {
-    my (@output) = @_;
-
-    verify_common (@output);
-    my (@t);
-    $t[4] = $t[5] = $t[6] = -1;
-    local ($_);
-    foreach (@output) {
-	my ($idx) = /^Thread (\d+)/ or next;
-	my ($iter) = /iteration (\d+)$/;
-	$iter = 5 if /done!$/;
-	die "Malformed output\n" if !defined $iter;
-	if ($idx == 6) {
-	    die "Thread 6 started before either other thread finished\n"
-		if $t[4] < 5 && $t[5] < 5;
-	    die "Thread 6 started before thread 4 finished\n"
-		if $t[4] < 5;
-	    die "Thread 6 started before thread 5 finished\n"
-		if $t[5] < 5;
-	}
-	die "Thread $idx out of order output\n" if $t[$idx] != $iter - 1;
-	$t[$idx] = $iter;
+    @output = fix_exit_codes (get_core_output (@output));
+    my ($n) = 0;
+    while (my ($m) = $output[0] =~ /^\(multi-oom\) begin (\d+)$/) {
+	die "Child process $m started out of order.\n" if $m != $n;
+	$n = $m + 1;
+	shift @output;
     }
-
-    my ($err) = "";
-    for my $idx (4, 5, 6) {
-	if ($t[$idx] == -1) {
-	    $err .= "Thread $idx did not run at all\n";
-	} elsif ($t[$idx] != 5) {
-	    $err .= "Thread $idx only completed $t[$idx] iterations\n";
-	}
+    die "Only $n child process(es) started.\n" if $n < 15;
+
+    # There could be a death notice for a process that didn't get
+    # fully loaded, and/or notices from the loader.
+    while (@output > 0
+	   && ($output[0] =~ /^multi-oom: exit\(-1\)$/
+	       || $output[0] =~ /^load: /)) {
+	shift @output;
     }
-    die $err if $err ne '';
-}
 
-sub grade_priority_fifo {
-    my (@output) = @_;
+    while (--$n >= 0) {
+	die "Output ended unexpectedly before process $n finished.\n"
+	    if @output < 2;
 
-    verify_common (@output);
-    my ($thread_cnt) = 10;
-    my ($iter_cnt) = 5;
-    my (@order);
-    my (@t) = (-1) x $thread_cnt;
-    local ($_);
-    foreach (@output) {
-	my ($idx) = /^Thread (\d+)/ or next;
-	my ($iter) = /iteration (\d+)$/;
-	$iter = $iter_cnt if /done!$/;
-	die "Malformed output\n" if !defined $iter;
-	if (@order < $thread_cnt) {
-	    push (@order, $idx);
-	    die "Thread $idx repeated within first $thread_cnt iterations: "
-		. join (' ', @order) . ".\n"
-		if grep ($_ == $idx, @order) != 1;
-	} else {
-	    die "Thread $idx ran when $order[0] should have.\n"
-		if $idx != $order[0];
-	    push (@order, shift @order);
-	}
-	die "Thread $idx out of order output.\n" if $t[$idx] != $iter - 1;
-	$t[$idx] = $iter;
-    }
+	local ($_);
+	chomp ($_ = shift @output);
+	die "Found '$_' expecting 'end' message.\n" if !/^\(multi-oom\) end/;
+	die "Child process $n ended out of order.\n"
+	    if !/^\(multi-oom\) end $n$/;
 
-    my ($err) = "";
-    for my $idx (0..$#t) {
-	if ($t[$idx] == -1) {
-	    $err .= "Thread $idx did not run at all.\n";
-	} elsif ($t[$idx] != $iter_cnt) {
-	    $err .= "Thread $idx only completed $t[$idx] iterations.\n";
-	}
+	chomp ($_ = shift @output);
+	die "Kernel didn't print proper exit message for process $n.\n"
+	    if !/^multi-oom: exit\($n\)$/;
     }
-    die $err if $err ne '';
-}
-
-sub grade_mlfqs_on {
-    my (@output) = @_;
-    verify_common (@output);
-    our (@mlfqs_on_stats) = mlfqs_stats (@output);
-}
-
-sub grade_mlfqs_off {
-    my (@output) = @_;
-    verify_common (@output);
-    our (@mlfqs_off_stats) = mlfqs_stats (@output);
-}
-
-sub grade_mlfqs_speedup {
-    our (@mlfqs_off_stats);
-    our (@mlfqs_on_stats);
-    eval {
-	check_mlfqs ();
-	my ($off_ticks) = $mlfqs_off_stats[1];
-	my ($on_ticks) = $mlfqs_on_stats[1];
-	die "$off_ticks ticks without MLFQS, $on_ticks with MLFQS\n"
-	    if $on_ticks >= $off_ticks;
-	die "ok\n";
-    };
-    chomp $@;
-    $result{'mlfqs-speedup'} = $@;
-}
-
-sub grade_mlfqs_priority {
-    our (@mlfqs_off_stats);
-    our (@mlfqs_on_stats);
-    eval {
-	check_mlfqs () if !defined (@mlfqs_on_stats);
-	for my $cat qw (CPU IO MIX) {
-	    die "Priority changed away from PRI_DEFAULT (29) without MLFQS\n"
-		if $mlfqs_off_stats[0]{$cat}{MIN} != 29
-		|| $mlfqs_off_stats[0]{$cat}{MAX} != 29;
-	    die "Minimum priority never changed from PRI_DEFAULT (29) "
-		. "with MLFQS\n"
-		if $mlfqs_on_stats[0]{$cat}{MIN} == 29;
-	    die "Maximum priority never changed from PRI_DEFAULT (29) "
-		. "with MLFQS\n"
-		if $mlfqs_on_stats[0]{$cat}{MAX} == 29;
-	}
-	die "ok\n";
-    };
-    chomp $@;
-    $result{'mlfqs-priority'} = $@;
+    die "Spurious output at end: '$output[0]'.\n" if @output;
 }
 
-sub check_mlfqs {
-    our (@mlfqs_off_stats);
-    our (@mlfqs_on_stats);
-    die "p1-4 didn't finish with MLFQS on or off\n"
-	if !defined (@mlfqs_off_stats) && !defined (@mlfqs_on_stats);
-    die "p1-4 didn't finish with MLFQS on\n"
-	if !defined (@mlfqs_on_stats);
-    die "p1-4 didn't finish with MLFQS off\n"
-	if !defined (@mlfqs_off_stats);
+sub get_file {
+    my ($guest_fn, $host_fn) = @_;
+    xsystem ("pintos "
+	     . "--os-disk=pintos/src/userprog/build/os.dsk "
+	     . "--fs-disk=output/$test/fs.dsk "
+	     . "-v get $guest_fn $host_fn",
+	     LOG => "$test/get-$guest_fn",
+	     TIMEOUT => 10)
+	or die "get $guest_fn failed\n";
 }
 
-sub mlfqs_stats {
-    my (@output) = @_;
-    my (%stats) = (CPU => {}, IO => {}, MIX => {});
-    my (%map) = ("CPU intensive" => 'CPU',
-		 "IO intensive" => 'IO',
-		 "Alternating IO/CPU" => 'MIX');
-    my (%rmap) = reverse %map;
-    my ($ticks);
-    local ($_);
-    foreach (@output) {
-	$ticks = $1 if /Timer: (\d+) ticks/;
-	my ($thread, $pri) = /^([A-Za-z\/ ]+): (\d+)$/ or next;
-	my ($t) = $map{$thread} or next;
-	
-	my ($s) = $stats{$t};
-	$$s{N}++;
-	$$s{SUM} += $pri;
-	$$s{SUM2} += $pri * $pri;
-	$$s{MIN} = $pri if !defined ($$s{MIN}) || $pri < $$s{MIN};
-	$$s{MAX} = $pri if !defined ($$s{MAX}) || $pri > $$s{MAX};
-    }
-
-    my (%expect_n) = (CPU => 5000, IO => 1000, MIX => 12000);
-    for my $cat (values (%map)) {
-	my ($s) = $stats{$cat};
-	die "$rmap{$cat} printed $$s{N} times, not $expect_n{$cat}\n"
-	    if $$s{N} != $expect_n{$cat};
-	die "$rmap{$cat} priority dropped to $$s{MIN}, below PRI_MIN (0)\n"
-	    if $$s{MIN} < 0;
-	die "$rmap{$cat} priority rose to $$s{MAX}, above PRI_MAX (59)\n"
-	    if $$s{MAX} > 59;
-	$$s{MEAN} = $$s{SUM} / $$s{N};
-    }
-
-    return (\%stats, $ticks);
-}
 
 sub verify_common {
     my (@output) = @_;
@@ -537,7 +411,7 @@ sub verify_common {
 	    } else {
 		$A2L = "i386-elf-addr2line";
 	    }
-	    open (A2L, "$A2L -fe output/$test/kernel.o @addrs|");
+	    open (A2L, "$A2L -fe pintos/src/userprog/build/kernel.o @addrs|");
 	    for (;;) {
 		my ($function, $line);
 		last unless defined ($function = <A2L>);
@@ -551,6 +425,25 @@ sub verify_common {
 	die "Kernel panic.  Details at end of file.\n"
     }
 
+    if (grep (/Pintos booting/, @output) > 1) {
+	my ($details);
+
+	$details = "Pintos spontaneously rebooted during this test.\n";
+	$details .= "This is most often due to unhandled page faults.\n";
+	$details .= "Here's the output from the initial boot through the\n";
+	$details .= "first reboot:\n\n";
+
+	my ($i) = 0;
+	local ($_);
+	for (@output) {
+	    $details .= "  $_\n";
+	    last if /Pintos booting/ && ++$i > 1;
+	}
+	$details{$test} = $details;
+	die "Triple-fault caused spontaneous reboot(s).  "
+	    . "Details at end of file.\n";
+    }
+
     die "No output at all\n" if @output == 0;
     die "Didn't start up properly: no \"Pintos booting\" startup message\n"
 	if !grep (/Pintos booting with.*kB RAM\.\.\./, @output);
@@ -586,20 +479,35 @@ sub get_core_output {
     return @output[$first ... $last];
 }
 
-sub compare_output {
-    my ($exp, @actual) = @_;
-    @actual = get_core_output (map ("$_\n", @actual));
+sub fix_exit_codes {
+    my (@output) = @_;
 
     # Fix up lines that look like exit codes.
-    for my $i (0...$#actual) {
-	if (my ($process, $code)
-	    = $actual[$i] =~ /^([-a-zA-Z0-9 ]+):.*[ \(](-?\d+)\b\)?$/) {
+    # Exit codes are supposed to be printed in the form "process: exit(code)"
+    # but people get unfortunately creative with it.
+    for my $i (0...$#output) {
+	local ($_) = $output[$i];
+	
+	my ($process, $code);
+	if ((($process, $code) = /^([-a-zA-Z0-9 ]+):.*[ \(](-?\d+)\b\)?$/)
+	    || (($process, $code) = /^([-a-zA-Z0-9 ]+) exit\((-?\d+)\)$/)
+	    || (($process, $code)
+		= /^([-a-zA-Z0-9 ]+) \(.*\): exit\((-?\d+)\)$/)
+	    || (($process, $code) = /^([-a-zA-Z0-9 ]+):\( (-?\d+) \) $/)
+) {
 	    $process = substr ($process, 0, 15);
 	    $process =~ s/\s.*//;
-	    $actual[$i] = "$process: exit($code)\n";
+	    $output[$i] = "$process: exit($code)\n";
 	}
     }
 
+    return @output;
+}
+
+sub compare_output {
+    my ($exp, @actual) = @_;
+    @actual = fix_exit_codes (get_core_output (map ("$_\n", @actual)));
+
     my ($details) = "";
     $details .= "$test actual output:\n";
     $details .= join ('', map ("  $_", @actual));
@@ -648,7 +556,22 @@ sub compare_output {
 
 	$details .= "Differences in `diff -u' format:\n";
 	$details .= join ('', @diff);
-	$details .= "(This is considered a `fuzzy match'.)\n" if $fuzzy_match;
+	$details .= "(This is considered a `fuzzy match'.)\n"
+	    if !$not_fuzzy_match;
+    }
+
+    if ($fuzzy_match) {
+	$details =
+	    "This test passed, but with extra, unexpected output.\n"
+	    . "Please inspect your code to make sure that it does not\n"
+	    . "produce output other than as specified in the project\n"
+	    . "description.\n\n"
+	    . "$details";
+    } else {
+	$details =
+	    "This test failed because its output did not match any\n"
+	    . "of the acceptable form(s).\n\n"
+	    . "$details";
     }
 
     $details{$test} = $details;
@@ -662,14 +585,26 @@ sub write_grades {
     my ($ploss) = 0;
     my ($tloss) = 0;
     my ($total) = 0;
+    my ($warnings) = 0;
     for (my ($i) = 0; $i <= $#summary; $i++) {
 	local ($_) = $summary[$i];
 	if (my ($loss, $test) = /^  -(\d+) ([-a-zA-Z0-9]+):/) {
 	    my ($result) = $result{$test} || "Not tested.";
 
 	    if ($result eq 'ok') {
-		splice (@summary, $i, 1);
-		$i--;
+		if (!defined $details{$test}) {
+		    # Test successful and no warnings.
+		    splice (@summary, $i, 1);
+		    $i--;
+		} else {
+		    # Test successful with warnings.
+		    s/-(\d+) //;
+		    $summary[$i] = $_;
+		    splice (@summary, $i + 1, 0,
+			    "    Test passed with warnings.  "
+			    . "Details at end of file.");
+		    $warnings++;
+		} 
 	    } else {
 		$ploss += $loss;
 		$tloss += $loss;
@@ -679,8 +614,10 @@ sub write_grades {
 	} elsif (my ($ptotal) = /^Score: \/(\d+)$/) {
 	    $total += $ptotal;
 	    $summary[$i] = "Score: " . ($ptotal - $ploss) . "/$ptotal";
-	    splice (@summary, $i, 0, "  All tests passed.") if $ploss == 0;
+	    splice (@summary, $i, 0, "  All tests passed.")
+		if $ploss == 0 && !$warnings;
 	    $ploss = 0;
+	    $warnings = 0;
 	    $i++;
 	}
     }
@@ -701,8 +638,15 @@ sub write_details {
 	my ($details) = $details{$test};
 	next if !defined ($details) && ! -e "output/$test/run.out";
 
+	my ($banner);
+	if ($result{$test} ne 'ok') {
+	    $banner = "$test failure details"; 
+	} else {
+	    $banner = "$test warnings";
+	}
+
 	print DETAILS "\n" if $n++;
-	print DETAILS "--- $test details ", '-' x (50 - length ($test));
+	print DETAILS "--- $banner ", '-' x (50 - length ($banner));
 	print DETAILS "\n\n";
 
 	if (!defined $details) {