From b9d904d1c1e432950f81e8cd8e4b5e8d134fc8e8 Mon Sep 17 00:00:00 2001 From: Ben Pfaff Date: Thu, 28 Oct 2004 01:00:02 +0000 Subject: [PATCH] Work on userprog testing. --- grading/threads/run-tests | 2 + grading/userprog/Makefile | 16 ++- grading/userprog/create-empty.c | 6 +- grading/userprog/create-empty.exp | 3 + grading/userprog/create-empty.exp1 | 2 + grading/userprog/create-normal.c | 6 +- grading/userprog/create-normal.exp | 4 + grading/userprog/create-null.c | 6 +- grading/userprog/create-null.exp | 3 + grading/userprog/create-null.exp1 | 2 + grading/userprog/panic.diff | 20 +++ grading/userprog/run-tests | 189 ++++++++++++++++------------- grading/userprog/tests.txt | 50 +++----- 13 files changed, 174 insertions(+), 135 deletions(-) create mode 100644 grading/userprog/create-empty.exp create mode 100644 grading/userprog/create-empty.exp1 create mode 100644 grading/userprog/create-normal.exp create mode 100644 grading/userprog/create-null.exp create mode 100644 grading/userprog/create-null.exp1 create mode 100644 grading/userprog/panic.diff diff --git a/grading/threads/run-tests b/grading/threads/run-tests index 94fd802..d5f5870 100755 --- a/grading/threads/run-tests +++ b/grading/threads/run-tests @@ -726,6 +726,8 @@ sub xsystem { unlink ("output/$log.err") if defined ($log) && $status == 0; + die $options{DIE} if $status != 0 && defined $options{DIE}; + return $status == 0; } diff --git a/grading/userprog/Makefile b/grading/userprog/Makefile index a928b97..f5917a6 100644 --- a/grading/userprog/Makefile +++ b/grading/userprog/Makefile @@ -13,15 +13,13 @@ DISKS = $(patsubst %,%.dsk,$(PROGS)) disks: $(DISKS) PINTOS = ../../src/utils/pintos -%.dsk: % os.dsk - rm -f $@ fs.dsk - $(PINTOS) make-disk fs.dsk 2 - $(PINTOS) -v run -f -q - $(PINTOS) -v put $< - mv fs.dsk $@ - -os.dsk: ../../src/userprog/build/os.dsk - ln -sf $< $@ +OS_DISK = ../../src/userprog/build/os.dsk +%.dsk: % + rm -f $@.tmp + $(PINTOS) make-disk $@.tmp 2 + $(PINTOS) -v --os-disk=$(OS_DISK) --fs-disk=$@.tmp run -f -q + $(PINTOS) -v --os-disk=$(OS_DISK) --fs-disk=$@.tmp put $< + mv $@.tmp $@ clean:: rm -f $(DISKS) diff --git a/grading/userprog/create-empty.c b/grading/userprog/create-empty.c index 3c4c0bd..4c4ffa3 100644 --- a/grading/userprog/create-empty.c +++ b/grading/userprog/create-empty.c @@ -4,8 +4,8 @@ int main (void) { - printf ("create(\"\"):\n"); - printf ("%d\n", create ("", 0)); - printf ("survived\n"); + printf ("(create-empty) begin\n"); + create ("", 0); + printf ("(create-empty) end\n"); return 0; } diff --git a/grading/userprog/create-empty.exp b/grading/userprog/create-empty.exp new file mode 100644 index 0000000..cba6c7a --- /dev/null +++ b/grading/userprog/create-empty.exp @@ -0,0 +1,3 @@ +(create-empty) begin +(create-empty) end +create-empty: exit(0) diff --git a/grading/userprog/create-empty.exp1 b/grading/userprog/create-empty.exp1 new file mode 100644 index 0000000..aac55e4 --- /dev/null +++ b/grading/userprog/create-empty.exp1 @@ -0,0 +1,2 @@ +(create-empty) begin +create-empty: exit(-1) diff --git a/grading/userprog/create-normal.c b/grading/userprog/create-normal.c index b483c9d..e2646b2 100644 --- a/grading/userprog/create-normal.c +++ b/grading/userprog/create-normal.c @@ -4,8 +4,8 @@ int main (void) { - printf ("create(\"quux.dat\"): "); - printf ("%d\n", create ("quux.dat", 0)); - printf ("survived\n"); + printf ("(create-normal) begin\n"); + printf ("(create-normal) create(): %d\n", create ("quux.dat", 0)); + printf ("(create-normal) end\n"); return 0; } diff --git a/grading/userprog/create-normal.exp b/grading/userprog/create-normal.exp new file mode 100644 index 0000000..93b8b48 --- /dev/null +++ b/grading/userprog/create-normal.exp @@ -0,0 +1,4 @@ +(create-normal) begin +(create-normal) create(): 1 +(create-normal) end +create-normal: exit(0) diff --git a/grading/userprog/create-null.c b/grading/userprog/create-null.c index e19d218..eafcb40 100644 --- a/grading/userprog/create-null.c +++ b/grading/userprog/create-null.c @@ -4,8 +4,8 @@ int main (void) { - printf ("create(null): "); - printf ("%d\n", create (NULL, 0)); - printf ("survived\n"); + printf ("(create-null) begin\n"); + create (NULL, 0); + printf ("(create-null) end\n"); return 0; } diff --git a/grading/userprog/create-null.exp b/grading/userprog/create-null.exp new file mode 100644 index 0000000..2250356 --- /dev/null +++ b/grading/userprog/create-null.exp @@ -0,0 +1,3 @@ +(create-null) begin +(create-null) end +create-null: exit(0) diff --git a/grading/userprog/create-null.exp1 b/grading/userprog/create-null.exp1 new file mode 100644 index 0000000..950d56f --- /dev/null +++ b/grading/userprog/create-null.exp1 @@ -0,0 +1,2 @@ +(create-null) begin +create-null: exit(-1) diff --git a/grading/userprog/panic.diff b/grading/userprog/panic.diff new file mode 100644 index 0000000..6134d47 --- /dev/null +++ b/grading/userprog/panic.diff @@ -0,0 +1,20 @@ +diff -up /home/blp/cs140/pintos/src/lib/debug.c.\~1.8.\~ /home/blp/cs140/pintos/src/lib/debug.c +--- /home/blp/cs140/pintos/src/lib/debug.c.~1.8.~ 2004-09-12 13:14:11.000000000 -0700 ++++ /home/blp/cs140/pintos/src/lib/debug.c 2004-10-17 00:02:32.000000000 -0700 +@@ -5,6 +5,7 @@ + #include + #include + #ifdef KERNEL ++#include "threads/init.h" + #include "threads/interrupt.h" + #include "devices/serial.h" + #else +@@ -83,7 +84,7 @@ debug_panic (const char *file, int line, + + #ifdef KERNEL + serial_flush (); +- for (;;); ++ power_off (); + #else + exit (1); + #endif diff --git a/grading/userprog/run-tests b/grading/userprog/run-tests index 62d243c..ddc5b6e 100755 --- a/grading/userprog/run-tests +++ b/grading/userprog/run-tests @@ -37,13 +37,8 @@ sub usage { } # Default set of tests. -@TESTS = ("alarm-single", "alarm-multiple", "alarm-zero", "alarm-negative", - "join-simple", - "join-quick", "join-multiple", "join-nested", - "join-dummy", "join-invalid", "join-no", - "priority-preempt", "priority-fifo", "priority-donate-one", - "priority-donate-multiple", "priority-donate-nest", - "mlfqs-on", "mlfqs-off") +@TESTS = qw (create-normal create-empty create-null + create-long) unless @TESTS > 0; # Handle final grade mode. @@ -118,6 +113,9 @@ if ($clean) { # Extract submission. extract_tarball () if ! -d "pintos"; +# Compile submission. +compile (); + # Verify that the proper directory was submitted. -d "pintos/src/threads" or die "pintos/src/threads: stat: $!\n"; @@ -133,15 +131,13 @@ for $test (@TESTS) { $result = grade_test ($test); $result =~ s/\n$//; } - print "$result\n"; + print "$result"; + print " - with warnings" if $result eq 'ok' && defined $details{$test}; + print "\n"; $result{$test} = $result; } -# MLFQS takes results from mlfqs-on and mlfqs-off. -grade_mlfqs_speedup (); -grade_mlfqs_priority (); - # Write output. write_grades (); write_details (); @@ -189,14 +185,12 @@ sub ext_mdyHMS { sub test_source { my ($test) = @_; my ($src) = "$GRADES_DIR/$test.c"; - $src = "$GRADES_DIR/mlfqs.c" if $test =~ /^mlfqs/; -e $src or die "$src: stat: $!\n"; return $src; } sub test_constants { my ($defines) = ""; - $defines .= "#define MLFQS 1\n" if $test eq 'mlfqs-on'; return $defines; } @@ -223,6 +217,12 @@ sub run_test { return $status; } +sub compile { + print "Compiling...\n"; + xsystem ("cd pintos/src/userprog && make", LOG => "make") + or return "compile error"; +} + sub really_run_test { # Need to run it. # If there's residue from an earlier test, move it to .old. @@ -233,46 +233,15 @@ sub really_run_test { # Make output directory. mkdir "output/$test"; - - # Change constants.h if necessary. - my ($defines) = test_constants ($test); - if ($defines ne snarf ("pintos/src/constants.h")) { - open (CONSTANTS, ">pintos/src/constants.h"); - print CONSTANTS $defines; - close (CONSTANTS); - } - - # Changes devices/timer.c if necessary. - my ($new_time_slice) = $test eq 'priority-fifo' ? 100 : 1; - my (@timer) = snarf ("pintos/src/devices/timer.c"); - if (!grep (/^\#define TIME_SLICE $new_time_slice$/, @timer)) { - @timer = grep (!/^\#define TIME_SLICE/, @timer); - unshift (@timer, "#define TIME_SLICE $new_time_slice"); - open (TIMER, ">pintos/src/devices/timer.c"); - print TIMER map ("$_\n", @timer); - close (TIMER); - } - - # Copy in the new test.c and delete enough files to ensure a full rebuild. - my ($src) = test_source ($test); - xsystem ("cp $src pintos/src/threads/test.c", DIE => "cp failed\n"); - unlink ("pintos/src/threads/build/threads/test.o"); - unlink ("pintos/src/threads/build/kernel.o"); - unlink ("pintos/src/threads/build/kernel.bin"); - unlink ("pintos/src/threads/build/os.dsk"); - - # Build. - xsystem ("cd pintos/src/threads && make", LOG => "$test/make") - or return "compile error"; - - # Copy out files for backtraces later. - xsystem ("cp pintos/src/threads/build/kernel.o output/$test"); - xsystem ("cp pintos/src/threads/build/os.dsk output/$test"); + xsystem ("cp $GRADES_DIR/$test.dsk output/$test/fs.dsk", + DIE => "cp failed\n"); # Run. my ($timeout) = 10; - $timeout = 600 if $test =~ /^mlfqs/; - xsystem ("cd pintos/src/threads/build && pintos -v run -q", + xsystem ("pintos " + . "--os-disk=pintos/src/userprog/build/os.dsk " + . "--fs-disk=output/$test/fs.dsk " + . "-v run -q -ex \"$test\"", LOG => "$test/run", TIMEOUT => $timeout) or return "Bochs error"; @@ -302,8 +271,11 @@ sub grade_test { return "ok"; } -sub grade_alarm_single { - verify_alarm (1, @_); +sub grade_create_empty { + my (@output) = @_; + verify_common (@_); + compare_output (["(create-empty) begin"], @output); + } sub grade_alarm_multiple { @@ -580,54 +552,95 @@ sub verify_common { if !grep (/Powering off/, @output); } -sub compare_output { - my ($exp_file, @actual) = @_; - my (@expected) = snarf ($exp_file); +sub eq_lines { + my ($actual, $expected) = @_; + return $actual eq $expected; +} +sub compare_output { + my ($exp, @actual) = @_; @actual = map ("$_\n", @actual); - @expected = map ("$_\n", @expected); # Trim header and trailer from @actual. - while (scalar (@actual) && $actual[0] ne $expected[0]) { - shift (@actual); + our ($test); + my ($first); + for ($first = 0; $first <= $#actual; $first++) { + $first++, last if $actual[$first] =~ /^Executing '$test':$/; } - die "First line of expected output was not present.\n" if !@actual; - while (scalar (@actual) && $actual[$#actual] ne $expected[$#expected]) { - pop (@actual); + + my ($last); + for ($last = $#actual; $last >= 0; $last--) { + $last--, last if $actual[$last] =~ /^Timer: \d+ ticks$/; } - die "Final line of expected output was not present.\n" if !@actual; - - # Check whether they're the same. - if ($#actual == $#expected) { - my ($eq) = 1; - for (my ($i) = 0; $i <= $#expected; $i++) { - $eq = 0 if $actual[$i] ne $expected[$i]; - } - return if $eq; + + if ($last < $first) { + my ($no_first) = $first > $#actual; + my ($no_last) = $last < $#actual; + die "Couldn't locate output.\n"; } - # They differ. Output a diff. - my (@diff) = ""; - my ($d) = Algorithm::Diff->new (\@expected, \@actual); - while ($d->Next ()) { - my ($ef, $el, $af, $al) = $d->Get (qw (min1 max1 min2 max2)); - if ($d->Same ()) { - push (@diff, map (" $_", $d->Items (1))); - } else { - push (@diff, map ("- $_", $d->Items (1))) if $d->Items (1); - push (@diff, map ("+ $_", $d->Items (2))) if $d->Items (2); + @actual = @actual[$first ... $last]; + + # Fix up lines that look like exit codes. + for my $i (0...$#actual) { + if (my ($process, $code) + = $actual[$i] =~ /^([-a-zA-Z0-9 ]+):.*[ \(](-?\d+)\b\)?$/) { + $process = substr ($process, 0, 15); + $actual[$i] = "$process: exit($code)\n"; } } my ($details) = ""; $details .= "$test actual output:\n"; $details .= join ('', map (" $_", @actual)); - $details .= "\n$test expected output:\n"; - $details .= join ('', map (" $_", @expected)); - $details .= "\nOutput differences in `diff -u' format:\n"; - $details .= join ('', @diff) . "\n"; + + my ($fuzzy_match) = 0; + for (my ($i) = 0; ; $i++) { + my ($fn) = $exp; + $fn .= $i if $i; + if (! -e $fn) { + die "$exp: stat: $!\n" if !$i; + last; + } + my (@expected) = map ("$_\n", snarf ($fn)); + + $details .= "\n$test acceptable output:\n"; + $details .= join ('', map (" $_", @expected)); + + # Check whether they're the same. + if ($#actual == $#expected) { + my ($eq) = 1; + for (my ($i) = 0; $i <= $#expected; $i++) { + $eq = 0 if !eq_lines ($actual[$i], $expected[$i]); + } + return if $eq; + } + + # They differ. Output a diff. + my (@diff) = ""; + my ($d) = Algorithm::Diff->new (\@expected, \@actual); + my ($all_additions) = 1; + while ($d->Next ()) { + my ($ef, $el, $af, $al) = $d->Get (qw (min1 max1 min2 max2)); + if ($d->Same ()) { + push (@diff, map (" $_", $d->Items (1))); + } else { + push (@diff, map ("- $_", $d->Items (1))) if $d->Items (1); + push (@diff, map ("+ $_", $d->Items (2))) if $d->Items (2); + $all_additions = 0 if $d->Items (1); + } + } + + $fuzzy_match = 1 if $all_additions; + + $details .= "Differences in `diff -u' format:\n"; + $details .= join ('', @diff); + $details .= "(This is considered a `fuzzy match'.)\n" if $fuzzy_match; + } + $details{$test} = $details; - die "Output differs from expected. Details at end of file.\n"; + die "Output differs from expected. Details at end of file.\n" + unless $fuzzy_match; } sub write_grades { @@ -670,7 +683,7 @@ sub write_details { open (DETAILS, ">details.out"); my ($n) = 0; for my $test (@TESTS) { - next if $result{$test} eq 'ok'; + next if $result{$test} eq 'ok' && !defined $details{$test}; my ($details) = $details{$test}; next if !defined ($details) && ! -e "output/$test/run.out"; @@ -726,6 +739,8 @@ sub xsystem { unlink ("output/$log.err") if defined ($log) && $status == 0; + die $options{DIE} if $status != 0 && defined $options{DIE}; + return $status == 0; } diff --git a/grading/userprog/tests.txt b/grading/userprog/tests.txt index 794fad1..1ba870e 100644 --- a/grading/userprog/tests.txt +++ b/grading/userprog/tests.txt @@ -2,10 +2,17 @@ CORRECTNESS [[total]] --------------------- Argument passing - -5 args-argc: argc is not set correctly - -5 args-argv0: executable name not passed as argv[0] - -5 args-multiple: passing multiple arguments fails - -5 args-dblspace: using multiple spaces between arguments fails + -3 args-argc: argc is not set correctly + -3 args-argv0: executable name not passed as argv[0] + -3 args-multiple: passing multiple arguments fails + -3 args-dblspace: using multiple spaces between arguments fails +Score: /12 + +System calls + -3 syscall-bad-sp: system call with a bad stack pointer must not crash OS + -3 syscall-bad-arg: syscall with argument off top of stack must not crash OS + -3 syscall-boundary: syscall with args across page boundary must work +Score: /9 System calls: halt, exec -1 halt: halt system call fails @@ -19,11 +26,13 @@ System calls: create -1 create-bad-ptr: pass invalid pointer to create system call -1 create-long: pass long file name to create system call -1 create-exists: pass name of an existing file to create system call + -1 create-boundary: pass name of file crossing page boundary Score: /7 System calls: open -2 open-normal: open a file in the most normal way -2 open-missing: try to open a file that doesn't exist + -2 open-boundary: pass name of file crossing page boundary -1 open-empty: pass empty string to open system call -1 open-null: pass null pointer to open system call -1 open-twice: open the same file twice @@ -40,6 +49,7 @@ Score: /7 System calls: read -2 read-normal: read from open file in most normal way -2 read-bad-ptr: pass invalid pointer to read system call + -2 read-boundary: pass buffer crossing page boundary -1 read-zero: try to read zero bytes -1 read-stdout: try to read from stdout -1 read-bad-fd: try to read from invalid file descriptor @@ -48,6 +58,7 @@ Score: /7 System calls: write -2 write-normal: write to open file in most normal way -2 write-bad-ptr: pass invalid pointer to write system call + -2 write-boundary: pass buffer crossing page boundary -1 write-zero: try to write zero bytes -1 write-stdin: try to write to stdin -1 write-bad-fd: try to write to invalid file descriptor @@ -68,29 +79,8 @@ System calls: join Score: /7 Multiprogramming - - - -3. Multiprogramming / Page boundary checking (/50 points) - Multiprogramming: Exec'ing two simple programs -30 - from another program doesn't work - Open/Create fails on page boundary -5 - Read/Write fails on page boundary -5 - Can close file opened by parent -5 - Running out of physical memory crashes OS -5 - Shell test doesn't work with normal page size -5 - Shell test doesn't work with smaller page size -5 - Exec/Join tests 0 and 1 fail when we substitute a new - bitmap.cc that returns pages in a random order. -5 - ------------ -Notes on the shell tests : - -Large page size echo/matmult test - -5 if nothing works no matter what you do - -4 if shell works in either case but matmult doesn't work in either case - -Small page size echo/matmult test - -5 if nothing works no matter what you do - -4 if shell works in either case but matmult doesn't work in either case ------------- - + -2 multi-recurse: test recursively executing subprocesses + -2 multi-oom: exhausting user memory must not crash OS + -2 multi-fragment: programs must load even if user memory is fragmented + -2 multi-parent-fd: child must not be able to close parent's fds +Score: /8 -- 2.30.2