From: Ben Pfaff <blp@cs.stanford.edu>
Date: Thu, 28 Oct 2004 01:00:02 +0000 (+0000)
Subject: Work on userprog testing.
X-Git-Url: https://pintos-os.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=b9d904d1c1e432950f81e8cd8e4b5e8d134fc8e8;p=pintos-anon

Work on userprog testing.
---

diff --git a/grading/threads/run-tests b/grading/threads/run-tests
index 94fd802..d5f5870 100755
--- a/grading/threads/run-tests
+++ b/grading/threads/run-tests
@@ -726,6 +726,8 @@ sub xsystem {
 
     unlink ("output/$log.err") if defined ($log) && $status == 0;
 
+    die $options{DIE} if $status != 0 && defined $options{DIE};
+
     return $status == 0;
 }
 
diff --git a/grading/userprog/Makefile b/grading/userprog/Makefile
index a928b97..f5917a6 100644
--- a/grading/userprog/Makefile
+++ b/grading/userprog/Makefile
@@ -13,15 +13,13 @@ DISKS = $(patsubst %,%.dsk,$(PROGS))
 disks: $(DISKS)
 
 PINTOS = ../../src/utils/pintos
-%.dsk: % os.dsk
-	rm -f $@ fs.dsk
-	$(PINTOS) make-disk fs.dsk 2
-	$(PINTOS) -v run -f -q
-	$(PINTOS) -v put $<
-	mv fs.dsk $@
-
-os.dsk: ../../src/userprog/build/os.dsk
-	ln -sf $< $@
+OS_DISK = ../../src/userprog/build/os.dsk
+%.dsk: %
+	rm -f $@.tmp
+	$(PINTOS) make-disk $@.tmp 2
+	$(PINTOS) -v --os-disk=$(OS_DISK) --fs-disk=$@.tmp run -f -q
+	$(PINTOS) -v --os-disk=$(OS_DISK) --fs-disk=$@.tmp put $<
+	mv $@.tmp $@
 
 clean::
 	rm -f $(DISKS)
diff --git a/grading/userprog/create-empty.c b/grading/userprog/create-empty.c
index 3c4c0bd..4c4ffa3 100644
--- a/grading/userprog/create-empty.c
+++ b/grading/userprog/create-empty.c
@@ -4,8 +4,8 @@
 int
 main (void) 
 {
-  printf ("create(\"\"):\n");
-  printf ("%d\n", create ("", 0));
-  printf ("survived\n");
+  printf ("(create-empty) begin\n");
+  create ("", 0);
+  printf ("(create-empty) end\n");
   return 0;
 }
diff --git a/grading/userprog/create-empty.exp b/grading/userprog/create-empty.exp
new file mode 100644
index 0000000..cba6c7a
--- /dev/null
+++ b/grading/userprog/create-empty.exp
@@ -0,0 +1,3 @@
+(create-empty) begin
+(create-empty) end
+create-empty: exit(0)
diff --git a/grading/userprog/create-empty.exp1 b/grading/userprog/create-empty.exp1
new file mode 100644
index 0000000..aac55e4
--- /dev/null
+++ b/grading/userprog/create-empty.exp1
@@ -0,0 +1,2 @@
+(create-empty) begin
+create-empty: exit(-1)
diff --git a/grading/userprog/create-normal.c b/grading/userprog/create-normal.c
index b483c9d..e2646b2 100644
--- a/grading/userprog/create-normal.c
+++ b/grading/userprog/create-normal.c
@@ -4,8 +4,8 @@
 int
 main (void) 
 {
-  printf ("create(\"quux.dat\"): ");
-  printf ("%d\n", create ("quux.dat", 0));
-  printf ("survived\n");
+  printf ("(create-normal) begin\n");
+  printf ("(create-normal) create(): %d\n", create ("quux.dat", 0));
+  printf ("(create-normal) end\n");
   return 0;
 }
diff --git a/grading/userprog/create-normal.exp b/grading/userprog/create-normal.exp
new file mode 100644
index 0000000..93b8b48
--- /dev/null
+++ b/grading/userprog/create-normal.exp
@@ -0,0 +1,4 @@
+(create-normal) begin
+(create-normal) create(): 1
+(create-normal) end
+create-normal: exit(0)
diff --git a/grading/userprog/create-null.c b/grading/userprog/create-null.c
index e19d218..eafcb40 100644
--- a/grading/userprog/create-null.c
+++ b/grading/userprog/create-null.c
@@ -4,8 +4,8 @@
 int
 main (void) 
 {
-  printf ("create(null): ");
-  printf ("%d\n", create (NULL, 0));
-  printf ("survived\n");
+  printf ("(create-null) begin\n");
+  create (NULL, 0);
+  printf ("(create-null) end\n");
   return 0;
 }
diff --git a/grading/userprog/create-null.exp b/grading/userprog/create-null.exp
new file mode 100644
index 0000000..2250356
--- /dev/null
+++ b/grading/userprog/create-null.exp
@@ -0,0 +1,3 @@
+(create-null) begin
+(create-null) end
+create-null: exit(0)
diff --git a/grading/userprog/create-null.exp1 b/grading/userprog/create-null.exp1
new file mode 100644
index 0000000..950d56f
--- /dev/null
+++ b/grading/userprog/create-null.exp1
@@ -0,0 +1,2 @@
+(create-null) begin
+create-null: exit(-1)
diff --git a/grading/userprog/panic.diff b/grading/userprog/panic.diff
new file mode 100644
index 0000000..6134d47
--- /dev/null
+++ b/grading/userprog/panic.diff
@@ -0,0 +1,20 @@
+diff -up /home/blp/cs140/pintos/src/lib/debug.c.\~1.8.\~ /home/blp/cs140/pintos/src/lib/debug.c
+--- /home/blp/cs140/pintos/src/lib/debug.c.~1.8.~	2004-09-12 13:14:11.000000000 -0700
++++ /home/blp/cs140/pintos/src/lib/debug.c	2004-10-17 00:02:32.000000000 -0700
+@@ -5,6 +5,7 @@
+ #include <stdio.h>
+ #include <string.h>
+ #ifdef KERNEL
++#include "threads/init.h"
+ #include "threads/interrupt.h"
+ #include "devices/serial.h"
+ #else
+@@ -83,7 +84,7 @@ debug_panic (const char *file, int line,
+ 
+ #ifdef KERNEL
+   serial_flush ();
+-  for (;;);
++  power_off ();
+ #else
+   exit (1);
+ #endif
diff --git a/grading/userprog/run-tests b/grading/userprog/run-tests
index 62d243c..ddc5b6e 100755
--- a/grading/userprog/run-tests
+++ b/grading/userprog/run-tests
@@ -37,13 +37,8 @@ sub usage {
 }
 
 # Default set of tests.
-@TESTS = ("alarm-single", "alarm-multiple", "alarm-zero", "alarm-negative",
-	  "join-simple",
-	  "join-quick", "join-multiple", "join-nested",
-	  "join-dummy", "join-invalid", "join-no",
-	  "priority-preempt", "priority-fifo", "priority-donate-one",
-	  "priority-donate-multiple", "priority-donate-nest",
-	  "mlfqs-on", "mlfqs-off")
+@TESTS = qw (create-normal create-empty create-null 
+	     create-long)
     unless @TESTS > 0;
 
 # Handle final grade mode.
@@ -118,6 +113,9 @@ if ($clean) {
 # Extract submission.
 extract_tarball () if ! -d "pintos";
 
+# Compile submission.
+compile ();
+
 # Verify that the proper directory was submitted.
 -d "pintos/src/threads" or die "pintos/src/threads: stat: $!\n";
 
@@ -133,15 +131,13 @@ for $test (@TESTS) {
 	$result = grade_test ($test);
 	$result =~ s/\n$//;
     }
-    print "$result\n";
+    print "$result";
+    print " - with warnings" if $result eq 'ok' && defined $details{$test};
+    print "\n";
     
     $result{$test} = $result;
 }
 
-# MLFQS takes results from mlfqs-on and mlfqs-off.
-grade_mlfqs_speedup ();
-grade_mlfqs_priority ();
-
 # Write output.
 write_grades ();
 write_details ();
@@ -189,14 +185,12 @@ sub ext_mdyHMS {
 sub test_source {
     my ($test) = @_;
     my ($src) = "$GRADES_DIR/$test.c";
-    $src = "$GRADES_DIR/mlfqs.c" if $test =~ /^mlfqs/;
     -e $src or die "$src: stat: $!\n";
     return $src;
 }
 
 sub test_constants {
    my ($defines) = "";
-   $defines .= "#define MLFQS 1\n" if $test eq 'mlfqs-on';
    return $defines;
  }
 
@@ -223,6 +217,12 @@ sub run_test {
     return $status;
 }
 
+sub compile {
+    print "Compiling...\n";
+    xsystem ("cd pintos/src/userprog && make", LOG => "make")
+	or return "compile error";
+}
+
 sub really_run_test {
     # Need to run it.
     # If there's residue from an earlier test, move it to .old.
@@ -233,46 +233,15 @@ sub really_run_test {
 
     # Make output directory.
     mkdir "output/$test";
-
-    # Change constants.h if necessary.
-    my ($defines) = test_constants ($test);
-    if ($defines ne snarf ("pintos/src/constants.h")) {
-	open (CONSTANTS, ">pintos/src/constants.h");
-	print CONSTANTS $defines;
-	close (CONSTANTS);
-    }
-
-    # Changes devices/timer.c if necessary.
-    my ($new_time_slice) = $test eq 'priority-fifo' ? 100 : 1;
-    my (@timer) = snarf ("pintos/src/devices/timer.c");
-    if (!grep (/^\#define TIME_SLICE $new_time_slice$/, @timer)) {
-	@timer = grep (!/^\#define TIME_SLICE/, @timer);
-	unshift (@timer, "#define TIME_SLICE $new_time_slice");
-	open (TIMER, ">pintos/src/devices/timer.c");
-	print TIMER map ("$_\n", @timer);
-	close (TIMER);
-    }
-
-    # Copy in the new test.c and delete enough files to ensure a full rebuild.
-    my ($src) = test_source ($test);
-    xsystem ("cp $src pintos/src/threads/test.c", DIE => "cp failed\n");
-    unlink ("pintos/src/threads/build/threads/test.o");
-    unlink ("pintos/src/threads/build/kernel.o");
-    unlink ("pintos/src/threads/build/kernel.bin");
-    unlink ("pintos/src/threads/build/os.dsk");
-
-    # Build.
-    xsystem ("cd pintos/src/threads && make", LOG => "$test/make")
-	or return "compile error";
-
-    # Copy out files for backtraces later.
-    xsystem ("cp pintos/src/threads/build/kernel.o output/$test");
-    xsystem ("cp pintos/src/threads/build/os.dsk output/$test");
+    xsystem ("cp $GRADES_DIR/$test.dsk output/$test/fs.dsk",
+	     DIE => "cp failed\n");
 
     # Run.
     my ($timeout) = 10;
-    $timeout = 600 if $test =~ /^mlfqs/;
-    xsystem ("cd pintos/src/threads/build && pintos -v run -q",
+    xsystem ("pintos "
+	     . "--os-disk=pintos/src/userprog/build/os.dsk "
+	     . "--fs-disk=output/$test/fs.dsk "
+	     . "-v run -q -ex \"$test\"",
 	     LOG => "$test/run",
 	     TIMEOUT => $timeout)
 	or return "Bochs error";
@@ -302,8 +271,11 @@ sub grade_test {
     return "ok";
 }
 
-sub grade_alarm_single {
-    verify_alarm (1, @_);
+sub grade_create_empty {
+    my (@output) = @_;
+    verify_common (@_);
+    compare_output (["(create-empty) begin"], @output);
+    
 }
 
 sub grade_alarm_multiple {
@@ -580,54 +552,95 @@ sub verify_common {
 	if !grep (/Powering off/, @output);
 }
 
-sub compare_output {
-    my ($exp_file, @actual) = @_;
-    my (@expected) = snarf ($exp_file);
+sub eq_lines {
+    my ($actual, $expected) = @_;
+    return $actual eq $expected;
+}
 
+sub compare_output {
+    my ($exp, @actual) = @_;
     @actual = map ("$_\n", @actual);
-    @expected = map ("$_\n", @expected);
 
     # Trim header and trailer from @actual.
-    while (scalar (@actual) && $actual[0] ne $expected[0]) {
-	shift (@actual);
+    our ($test);
+    my ($first);
+    for ($first = 0; $first <= $#actual; $first++) {
+	$first++, last if $actual[$first] =~ /^Executing '$test':$/;
     }
-    die "First line of expected output was not present.\n" if !@actual;
-    while (scalar (@actual) && $actual[$#actual] ne $expected[$#expected]) {
-	pop (@actual);
+
+    my ($last);
+    for ($last = $#actual; $last >= 0; $last--) {
+	$last--, last if $actual[$last] =~ /^Timer: \d+ ticks$/;
     }
-    die "Final line of expected output was not present.\n" if !@actual;
-    
-    # Check whether they're the same.
-    if ($#actual == $#expected) {
-	my ($eq) = 1;
-	for (my ($i) = 0; $i <= $#expected; $i++) {
-	    $eq = 0 if $actual[$i] ne $expected[$i];
-	}
-	return if $eq;
+
+    if ($last < $first) {
+	my ($no_first) = $first > $#actual;
+	my ($no_last) = $last < $#actual;
+	die "Couldn't locate output.\n";
     }
 
-    # They differ.  Output a diff.
-    my (@diff) = "";
-    my ($d) = Algorithm::Diff->new (\@expected, \@actual);
-    while ($d->Next ()) {
-	my ($ef, $el, $af, $al) = $d->Get (qw (min1 max1 min2 max2));
-	if ($d->Same ()) {
-	    push (@diff, map ("  $_", $d->Items (1)));
-	} else {
-	    push (@diff, map ("- $_", $d->Items (1))) if $d->Items (1);
-	    push (@diff, map ("+ $_", $d->Items (2))) if $d->Items (2);
+    @actual = @actual[$first ... $last];
+
+    # Fix up lines that look like exit codes.
+    for my $i (0...$#actual) {
+	if (my ($process, $code)
+	    = $actual[$i] =~ /^([-a-zA-Z0-9 ]+):.*[ \(](-?\d+)\b\)?$/) {
+	    $process = substr ($process, 0, 15);
+	    $actual[$i] = "$process: exit($code)\n";
 	}
     }
 
     my ($details) = "";
     $details .= "$test actual output:\n";
     $details .= join ('', map ("  $_", @actual));
-    $details .= "\n$test expected output:\n";
-    $details .= join ('', map ("  $_", @expected));
-    $details .= "\nOutput differences in `diff -u' format:\n";
-    $details .= join ('', @diff) . "\n";
+
+    my ($fuzzy_match) = 0;
+    for (my ($i) = 0; ; $i++) {
+	my ($fn) = $exp;
+	$fn .= $i if $i;
+	if (! -e $fn) {
+	    die "$exp: stat: $!\n" if !$i;
+	    last;
+	}
+	my (@expected) = map ("$_\n", snarf ($fn));
+
+	$details .= "\n$test acceptable output:\n";
+	$details .= join ('', map ("  $_", @expected));
+
+	# Check whether they're the same.
+	if ($#actual == $#expected) {
+	    my ($eq) = 1;
+	    for (my ($i) = 0; $i <= $#expected; $i++) {
+		$eq = 0 if !eq_lines ($actual[$i], $expected[$i]);
+	    }
+	    return if $eq;
+	}
+
+	# They differ.  Output a diff.
+	my (@diff) = "";
+	my ($d) = Algorithm::Diff->new (\@expected, \@actual);
+	my ($all_additions) = 1;
+	while ($d->Next ()) {
+	    my ($ef, $el, $af, $al) = $d->Get (qw (min1 max1 min2 max2));
+	    if ($d->Same ()) {
+		push (@diff, map ("  $_", $d->Items (1)));
+	    } else {
+		push (@diff, map ("- $_", $d->Items (1))) if $d->Items (1);
+		push (@diff, map ("+ $_", $d->Items (2))) if $d->Items (2);
+		$all_additions = 0 if $d->Items (1);
+	    }
+	}
+
+	$fuzzy_match = 1 if $all_additions;
+
+	$details .= "Differences in `diff -u' format:\n";
+	$details .= join ('', @diff);
+	$details .= "(This is considered a `fuzzy match'.)\n" if $fuzzy_match;
+    }
+
     $details{$test} = $details;
-    die "Output differs from expected.  Details at end of file.\n";
+    die "Output differs from expected.  Details at end of file.\n"
+	unless $fuzzy_match;
 }
 
 sub write_grades {
@@ -670,7 +683,7 @@ sub write_details {
     open (DETAILS, ">details.out");
     my ($n) = 0;
     for my $test (@TESTS) {
-	next if $result{$test} eq 'ok';
+	next if $result{$test} eq 'ok' && !defined $details{$test};
 	
 	my ($details) = $details{$test};
 	next if !defined ($details) && ! -e "output/$test/run.out";
@@ -726,6 +739,8 @@ sub xsystem {
 
     unlink ("output/$log.err") if defined ($log) && $status == 0;
 
+    die $options{DIE} if $status != 0 && defined $options{DIE};
+
     return $status == 0;
 }
 
diff --git a/grading/userprog/tests.txt b/grading/userprog/tests.txt
index 794fad1..1ba870e 100644
--- a/grading/userprog/tests.txt
+++ b/grading/userprog/tests.txt
@@ -2,10 +2,17 @@ CORRECTNESS [[total]]
 ---------------------
 
 Argument passing
-  -5 args-argc: argc is not set correctly
-  -5 args-argv0: executable name not passed as argv[0]
-  -5 args-multiple: passing multiple arguments fails
-  -5 args-dblspace: using multiple spaces between arguments fails
+  -3 args-argc: argc is not set correctly
+  -3 args-argv0: executable name not passed as argv[0]
+  -3 args-multiple: passing multiple arguments fails
+  -3 args-dblspace: using multiple spaces between arguments fails
+Score: /12
+
+System calls
+  -3 syscall-bad-sp: system call with a bad stack pointer must not crash OS
+  -3 syscall-bad-arg: syscall with argument off top of stack must not crash OS
+  -3 syscall-boundary: syscall with args across page boundary must work
+Score: /9
 
 System calls: halt, exec 
   -1 halt: halt system call fails
@@ -19,11 +26,13 @@ System calls: create
   -1 create-bad-ptr: pass invalid pointer to create system call
   -1 create-long: pass long file name to create system call
   -1 create-exists: pass name of an existing file to create system call
+  -1 create-boundary: pass name of file crossing page boundary
 Score: /7
 
 System calls: open
   -2 open-normal: open a file in the most normal way
   -2 open-missing: try to open a file that doesn't exist
+  -2 open-boundary: pass name of file crossing page boundary
   -1 open-empty: pass empty string to open system call
   -1 open-null: pass null pointer to open system call
   -1 open-twice: open the same file twice
@@ -40,6 +49,7 @@ Score: /7
 System calls: read
   -2 read-normal: read from open file in most normal way
   -2 read-bad-ptr: pass invalid pointer to read system call
+  -2 read-boundary: pass buffer crossing page boundary
   -1 read-zero: try to read zero bytes
   -1 read-stdout: try to read from stdout
   -1 read-bad-fd: try to read from invalid file descriptor
@@ -48,6 +58,7 @@ Score: /7
 System calls: write
   -2 write-normal: write to open file in most normal way
   -2 write-bad-ptr: pass invalid pointer to write system call
+  -2 write-boundary: pass buffer crossing page boundary
   -1 write-zero: try to write zero bytes
   -1 write-stdin: try to write to stdin
   -1 write-bad-fd: try to write to invalid file descriptor
@@ -68,29 +79,8 @@ System calls: join
 Score: /7
 
 Multiprogramming
-  -
-
-3. Multiprogramming / Page boundary checking (/50 points)
-        Multiprogramming: Exec'ing two simple programs                  -30 
-                          from another program doesn't work
-        Open/Create fails on page boundary                              -5
-        Read/Write fails on page boundary                               -5
-        Can close file opened by parent                                 -5
-        Running out of physical memory crashes OS                       -5
-        Shell test doesn't work with normal page size                   -5 
-        Shell test doesn't work with smaller page size                  -5
-        Exec/Join tests 0 and 1 fail when we substitute a new 
-        bitmap.cc that returns pages in a random order.                 -5 
-
------------
-Notes on the shell tests :
-
-Large page size echo/matmult test
-  -5 if nothing works no matter what you do
-  -4 if shell works in either case but matmult doesn't work in either case
-
-Small page size echo/matmult test
-  -5 if nothing works no matter what you do
-  -4 if shell works in either case but matmult doesn't work in either case
-------------
-
+  -2 multi-recurse: test recursively executing subprocesses
+  -2 multi-oom: exhausting user memory must not crash OS
+  -2 multi-fragment: programs must load even if user memory is fragmented
+  -2 multi-parent-fd: child must not be able to close parent's fds
+Score: /8