From: Ben Pfaff Date: Sun, 24 Aug 2008 04:31:40 +0000 (+0000) Subject: Use standard POSIX "ustar" format for the scratch disk. X-Git-Url: https://pintos-os.org/cgi-bin/gitweb.cgi?p=pintos-anon;a=commitdiff_plain;h=59f738d500f51ffc5f487344865b8bed69c26281 Use standard POSIX "ustar" format for the scratch disk. This should make it easier to work with scratch disks directly, instead of through the "pintos" script. With naming suggestion from Godmar Back. --- diff --git a/doc/bibliography.texi b/doc/bibliography.texi index 3597e20..e998a07 100644 --- a/doc/bibliography.texi +++ b/doc/bibliography.texi @@ -106,6 +106,10 @@ Edition}. 80@var{x}86-specific parts of the Unix interface. Interface---DRAFT---24 April 2001}. A draft of a revised version of @bibref{SysV-ABI} which was never completed. +@bibdfn{SUSv3} +The Open Group, @uref{http://www.unix.org/single_unix_specification/, +, Single UNIX Specification V3}, 2001. + @node Operating System Design References @section Operating System Design References diff --git a/doc/filesys.texi b/doc/filesys.texi index 2567d6a..eb2ba76 100644 --- a/doc/filesys.texi +++ b/doc/filesys.texi @@ -303,7 +303,7 @@ are straightforward once the above syscalls are implemented. We have also provided @command{pwd}, which is not so straightforward. The @command{shell} program implements @command{cd} internally. -The @code{pintos} @option{put} and @option{get} commands should now +The @code{pintos} @option{extract} and @option{append} commands should now accept full path names, assuming that the directories used in the paths have already been created. This should not require any significant extra effort on your part. diff --git a/doc/userprog.texi b/doc/userprog.texi index 612bb81..aaf75e2 100644 --- a/doc/userprog.texi +++ b/doc/userprog.texi @@ -204,7 +204,7 @@ commands for copying files out of a VM are similar, but substitute @option{-g} for @option{-p}. Incidentally, these commands work by passing special commands -@command{put} and @command{get} on the kernel's command line and copying +@command{extract} and @command{append} on the kernel's command line and copying to and from a special simulated ``scratch'' disk. If you're very curious, you can look at the @command{pintos} script as well as @file{filesys/fsutil.c} to learn the implementation details. diff --git a/src/Makefile.build b/src/Makefile.build index 0934551..4bd9220 100644 --- a/src/Makefile.build +++ b/src/Makefile.build @@ -38,7 +38,8 @@ lib_SRC += lib/random.c # Pseudo-random numbers. lib_SRC += lib/stdio.c # I/O library. lib_SRC += lib/stdlib.c # Utility functions. lib_SRC += lib/string.c # String functions. -lib_SRC += lib/arithmetic.c +lib_SRC += lib/arithmetic.c # 64-bit arithmetic for GCC. +lib_SRC += lib/ustar.c # Unix standard tar format utilities. # Kernel-specific library code. lib/kernel_SRC = lib/kernel/debug.c # Debug helpers. diff --git a/src/Makefile.userprog b/src/Makefile.userprog index f90b08c..0df391a 100644 --- a/src/Makefile.userprog +++ b/src/Makefile.userprog @@ -12,7 +12,8 @@ lib_SRC += lib/random.c # Pseudo-random numbers. lib_SRC += lib/stdio.c # I/O library. lib_SRC += lib/stdlib.c # Utility functions. lib_SRC += lib/string.c # String functions. -lib_SRC += lib/arithmetic.c +lib_SRC += lib/arithmetic.c # 64-bit arithmetic for GCC. +lib_SRC += lib/ustar.c # Unix standard tar format utilities. # User level only library code. lib/user_SRC = lib/user/debug.c # Debug helpers. diff --git a/src/filesys/fsutil.c b/src/filesys/fsutil.c index 14a4507..bf2c154 100644 --- a/src/filesys/fsutil.c +++ b/src/filesys/fsutil.c @@ -3,6 +3,7 @@ #include #include #include +#include #include "filesys/directory.h" #include "filesys/file.h" #include "filesys/filesys.h" @@ -66,85 +67,103 @@ fsutil_rm (char **argv) PANIC ("%s: delete failed\n", file_name); } -/* Copies from the "scratch" disk, hdc or hd1:0 to file ARGV[1] - in the file system. - - The current sector on the scratch disk must begin with the - string "PUT\0" followed by a 32-bit little-endian integer - indicating the file size in bytes. Subsequent sectors hold - the file content. - - The first call to this function will read starting at the - beginning of the scratch disk. Later calls advance across the - disk. This disk position is independent of that used for - fsutil_get(), so all `put's should precede all `get's. */ +/* Extracts a ustar-format tar archive from the scratch disk, hdc + or hd1:0, into the Pintos file system. */ void -fsutil_put (char **argv) +fsutil_extract (char **argv UNUSED) { static disk_sector_t sector = 0; - const char *file_name = argv[1]; struct disk *src; - struct file *dst; - off_t size; - void *buffer; + void *header, *data; - printf ("Putting '%s' into the file system...\n", file_name); + /* Allocate buffers. */ + header = malloc (DISK_SECTOR_SIZE); + data = malloc (DISK_SECTOR_SIZE); + if (header == NULL || data == NULL) + PANIC ("couldn't allocate buffers"); - /* Allocate buffer. */ - buffer = malloc (DISK_SECTOR_SIZE); - if (buffer == NULL) - PANIC ("couldn't allocate buffer"); - - /* Open source disk and read file size. */ + /* Open source disk. */ src = disk_get (1, 0); if (src == NULL) - PANIC ("couldn't open source disk (hdc or hd1:0)"); - - /* Read file size. */ - disk_read (src, sector++, buffer); - if (memcmp (buffer, "PUT", 4)) - PANIC ("%s: missing PUT signature on scratch disk", file_name); - size = ((int32_t *) buffer)[1]; - if (size < 0) - PANIC ("%s: invalid file size %d", file_name, size); - - /* Create destination file. */ - if (!filesys_create (file_name, size)) - PANIC ("%s: create failed", file_name); - dst = filesys_open (file_name); - if (dst == NULL) - PANIC ("%s: open failed", file_name); + PANIC ("couldn't open scratch disk (hdc or hd1:0)"); - /* Do copy. */ - while (size > 0) + printf ("Extracting ustar archive from scratch disk into file system...\n"); + + for (;;) { - int chunk_size = size > DISK_SECTOR_SIZE ? DISK_SECTOR_SIZE : size; - disk_read (src, sector++, buffer); - if (file_write (dst, buffer, chunk_size) != chunk_size) - PANIC ("%s: write failed with %"PROTd" bytes unwritten", - file_name, size); - size -= chunk_size; + const char *file_name; + const char *error; + enum ustar_type type; + int size; + + /* Read and parse ustar header. */ + disk_read (src, sector++, header); + error = ustar_parse_header (header, &file_name, &type, &size); + if (error != NULL) + PANIC ("bad ustar header in sector %"PRDSNu" (%s)", sector - 1, error); + + if (type == USTAR_EOF) + { + /* End of archive. */ + break; + } + else if (type == USTAR_DIRECTORY) + printf ("ignoring directory %s\n", file_name); + else if (type == USTAR_REGULAR) + { + struct file *dst; + + printf ("Putting '%s' into the file system...\n", file_name); + + /* Create destination file. */ + if (!filesys_create (file_name, size)) + PANIC ("%s: create failed", file_name); + dst = filesys_open (file_name); + if (dst == NULL) + PANIC ("%s: open failed", file_name); + + /* Do copy. */ + while (size > 0) + { + int chunk_size = (size > DISK_SECTOR_SIZE + ? DISK_SECTOR_SIZE + : size); + disk_read (src, sector++, data); + if (file_write (dst, data, chunk_size) != chunk_size) + PANIC ("%s: write failed with %d bytes unwritten", + file_name, size); + size -= chunk_size; + } + + /* Finish up. */ + file_close (dst); + } } - /* Finish up. */ - file_close (dst); - free (buffer); + /* Erase the ustar header from the start of the disk, so that + the extraction operation is idempotent. We erase two blocks + because two blocks of zeros are the ustar end-of-archive + marker. */ + printf ("Erasing ustar archive...\n"); + memset (header, 0, DISK_SECTOR_SIZE); + disk_write (src, 0, header); + disk_write (src, 1, header); + + free (data); + free (header); } -/* Copies file FILE_NAME from the file system to the scratch disk. - - The current sector on the scratch disk will receive "GET\0" - followed by the file's size in bytes as a 32-bit, - little-endian integer. Subsequent sectors receive the file's - data. +/* Copies file FILE_NAME from the file system to the scratch + disk, in ustar format. The first call to this function will write starting at the beginning of the scratch disk. Later calls advance across the - disk. This disk position is independent of that used for - fsutil_put(), so all `put's should precede all `get's. */ + disk. This position is independent of that used for + fsutil_extract(), so `extract' should precede all + `append's. */ void -fsutil_get (char **argv) +fsutil_append (char **argv) { static disk_sector_t sector = 0; @@ -154,7 +173,7 @@ fsutil_get (char **argv) struct disk *dst; off_t size; - printf ("Getting '%s' from the file system...\n", file_name); + printf ("Appending '%s' to ustar archive on scratch disk...\n", file_name); /* Allocate buffer. */ buffer = malloc (DISK_SECTOR_SIZE); @@ -172,12 +191,11 @@ fsutil_get (char **argv) if (dst == NULL) PANIC ("couldn't open target disk (hdc or hd1:0)"); - /* Write size to sector 0. */ - memset (buffer, 0, DISK_SECTOR_SIZE); - memcpy (buffer, "GET", 4); - ((int32_t *) buffer)[1] = size; + /* Write ustar header to first sector. */ + if (!ustar_make_header (file_name, USTAR_REGULAR, size, buffer)) + PANIC ("%s: name too long for ustar format", file_name); disk_write (dst, sector++, buffer); - + /* Do copy. */ while (size > 0) { @@ -191,6 +209,13 @@ fsutil_get (char **argv) size -= chunk_size; } + /* Write ustar end-of-archive marker, which is two consecutive + sectors full of zeros. Don't advance our position past + them, though, in case we have more files to append. */ + memset (buffer, 0, DISK_SECTOR_SIZE); + disk_write (dst, sector, buffer); + disk_write (dst, sector, buffer + 1); + /* Finish up. */ file_close (src); free (buffer); diff --git a/src/filesys/fsutil.h b/src/filesys/fsutil.h index abebfe2..cc73705 100644 --- a/src/filesys/fsutil.h +++ b/src/filesys/fsutil.h @@ -4,7 +4,7 @@ void fsutil_ls (char **argv); void fsutil_cat (char **argv); void fsutil_rm (char **argv); -void fsutil_put (char **argv); -void fsutil_get (char **argv); +void fsutil_extract (char **argv); +void fsutil_append (char **argv); #endif /* filesys/fsutil.h */ diff --git a/src/lib/packed.h b/src/lib/packed.h new file mode 100644 index 0000000..9a9b6e2 --- /dev/null +++ b/src/lib/packed.h @@ -0,0 +1,10 @@ +#ifndef __LIB_PACKED_H +#define __LIB_PACKED_H + +/* The "packed" attribute, when applied to a structure, prevents + GCC from inserting padding bytes between or after structure + members. It must be specified at the time of the structure's + definition, normally just after the closing brace. */ +#define PACKED __attribute__ ((packed)) + +#endif /* lib/packed.h */ diff --git a/src/lib/ustar.c b/src/lib/ustar.c new file mode 100644 index 0000000..49af69a --- /dev/null +++ b/src/lib/ustar.c @@ -0,0 +1,228 @@ +#include +#include +#include +#include +#include + +/* Header for ustar-format tar archive. See the documentation of + the "pax" utility in [SUSv3] for the the "ustar" format + specification. */ +struct ustar_header + { + char name[100]; /* File name. Null-terminated if room. */ + char mode[8]; /* Permissions as octal string. */ + char uid[8]; /* User ID as octal string. */ + char gid[8]; /* Group ID as octal string. */ + char size[12]; /* File size in bytes as octal string. */ + char mtime[12]; /* Modification time in seconds + from Jan 1, 1970, as octal string. */ + char chksum[8]; /* Sum of octets in header as octal string. */ + char typeflag; /* An enum ustar_type value. */ + char linkname[100]; /* Name of link target. + Null-terminated if room. */ + char magic[6]; /* "ustar\0" */ + char version[2]; /* "00" */ + char uname[32]; /* User name, always null-terminated. */ + char gname[32]; /* Group name, always null-terminated. */ + char devmajor[8]; /* Device major number as octal string. */ + char devminor[8]; /* Device minor number as octal string. */ + char prefix[155]; /* Prefix to file name. + Null-terminated if room. */ + char padding[12]; /* Pad to 512 bytes. */ + } +PACKED; + +/* Returns the checksum for the given ustar format HEADER. */ +static unsigned int +calculate_chksum (const struct ustar_header *h) +{ + const uint8_t *header = (const uint8_t *) h; + unsigned int chksum; + size_t i; + + chksum = 0; + for (i = 0; i < USTAR_HEADER_SIZE; i++) + { + /* The ustar checksum is calculated as if the chksum field + were all spaces. */ + const size_t chksum_start = offsetof (struct ustar_header, chksum); + const size_t chksum_end = chksum_start + sizeof h->chksum; + bool in_chksum_field = i >= chksum_start && i < chksum_end; + chksum += in_chksum_field ? ' ' : header[i]; + } + return chksum; +} + +/* Drop possibly dangerous prefixes from FILE_NAME and return the + stripped name. An archive with file names that start with "/" + or "../" could cause a naive tar extractor to write to + arbitrary parts of the file system, not just the destination + directory. We don't want to create such archives or be such a + naive extractor. + + The return value can be a suffix of FILE_NAME or a string + literal. */ +static const char * +strip_antisocial_prefixes (const char *file_name) +{ + while (*file_name == '/' + || !memcmp (file_name, "./", 2) + || !memcmp (file_name, "../", 3)) + file_name = strchr (file_name, '/') + 1; + return *file_name == '\0' || !strcmp (file_name, "..") ? "." : file_name; +} + +/* Composes HEADER as a USTAR_HEADER_SIZE (512)-byte archive + header in ustar format for a SIZE-byte file named FILE_NAME of + the given TYPE. The caller is responsible for writing the + header to a file or device. + + If successful, returns true. On failure (due to an + excessively long file name), returns false. */ +bool +ustar_make_header (const char *file_name, enum ustar_type type, + int size, char header[USTAR_HEADER_SIZE]) +{ + struct ustar_header *h = (struct ustar_header *) header; + + ASSERT (sizeof (struct ustar_header) == USTAR_HEADER_SIZE); + ASSERT (type == USTAR_REGULAR || type == USTAR_DIRECTORY); + + /* Check file name. */ + file_name = strip_antisocial_prefixes (file_name); + if (strlen (file_name) > 99) + { + printf ("%s: file name too long\n", file_name); + return false; + } + + /* Fill in header except for final checksum. */ + memset (h, 0, sizeof *h); + strlcpy (h->name, file_name, sizeof h->name); + snprintf (h->mode, sizeof h->mode, "%07o", + type == USTAR_REGULAR ? 0644 : 0755); + strlcpy (h->uid, "0000000", sizeof h->uid); + strlcpy (h->gid, "0000000", sizeof h->gid); + snprintf (h->size, sizeof h->size, "%011o", size); + snprintf (h->mtime, sizeof h->size, "%011o", 1136102400); + h->typeflag = type; + strlcpy (h->magic, "ustar", sizeof h->magic); + h->version[0] = h->version[1] = '0'; + strlcpy (h->gname, "root", sizeof h->gname); + strlcpy (h->uname, "root", sizeof h->uname); + + /* Compute and fill in final checksum. */ + snprintf (h->chksum, sizeof h->chksum, "%07o", calculate_chksum (h)); + + return true; +} + +/* Parses a SIZE-byte octal field in S in the format used by + ustar format. If successful, stores the field's value in + *VALUE and returns true; on failure, returns false. + + ustar octal fields consist of a sequence of octal digits + terminated by a space or a null byte. The ustar specification + seems ambiguous as to whether these fields must be padded on + the left with '0's, so we accept any field that fits in the + available space, regardless of whether it fills the space. */ +static bool +parse_octal_field (const char *s, size_t size, unsigned long int *value) +{ + size_t ofs; + + *value = 0; + for (ofs = 0; ofs < size; ofs++) + { + char c = s[ofs]; + if (c >= '0' && c <= '7') + { + if (*value > ULONG_MAX / 8) + { + /* Overflow. */ + return false; + } + *value = c - '0' + *value * 8; + } + else if (c == ' ' || c == '\0') + { + /* End of field, but disallow completely empty + fields. */ + return ofs > 0; + } + else + { + /* Bad character. */ + return false; + } + } + + /* Field did not end in space or null byte. */ + return false; +} + +/* Returns true if the CNT bytes starting at BLOCK are all zero, + false otherwise. */ +static bool +is_all_zeros (const char *block, size_t cnt) +{ + while (cnt-- > 0) + if (*block++ != 0) + return false; + return true; +} + +/* Parses HEADER as a ustar-format archive header for a regular + file or directory. If successful, stores the archived file's + name in *FILE_NAME (as a pointer into HEADER or a string + literal), its type in *TYPE, and its size in bytes in *SIZE, + and returns a null pointer. On failure, returns a + human-readable error message. */ +const char * +ustar_parse_header (const char header[USTAR_HEADER_SIZE], + const char **file_name, enum ustar_type *type, int *size) +{ + const struct ustar_header *h = (const struct ustar_header *) header; + unsigned long int chksum, size_ul; + + ASSERT (sizeof (struct ustar_header) == USTAR_HEADER_SIZE); + + /* Detect end of archive. */ + if (is_all_zeros (header, USTAR_HEADER_SIZE)) + { + *file_name = NULL; + *type = USTAR_EOF; + *size = 0; + return NULL; + } + + /* Validate ustar header. */ + if (memcmp (h->magic, "ustar", 6)) + return "not a ustar archive"; + else if (h->version[0] != '0' || h->version[1] != '0') + return "invalid ustar version"; + else if (!parse_octal_field (h->chksum, sizeof h->chksum, &chksum)) + return "corrupt chksum field"; + else if (chksum != calculate_chksum (h)) + return "checksum mismatch"; + else if (h->name[sizeof h->name - 1] != '\0' || h->prefix[0] != '\0') + return "file name too long"; + else if (h->typeflag != USTAR_REGULAR && h->typeflag != USTAR_DIRECTORY) + return "unimplemented file type"; + if (h->typeflag == USTAR_REGULAR) + { + if (!parse_octal_field (h->size, sizeof h->size, &size_ul)) + return "corrupt file size field"; + else if (size_ul > INT_MAX) + return "file too large"; + } + else + size_ul = 0; + + /* Success. */ + *file_name = strip_antisocial_prefixes (h->name); + *type = h->typeflag; + *size = size_ul; + return NULL; +} + diff --git a/src/lib/ustar.h b/src/lib/ustar.h new file mode 100644 index 0000000..43a5513 --- /dev/null +++ b/src/lib/ustar.h @@ -0,0 +1,29 @@ +#ifndef __LIB_USTAR_H +#define __LIB_USTAR_H + +/* Support for the standard Posix "ustar" format. See the + documentation of the "pax" utility in [SUSv3] for the the + "ustar" format specification. */ + +#include + +/* Type of a file entry in an archive. + The values here are the bytes that appear in the file format. + Only types of interest to Pintos are listed here. */ +enum ustar_type + { + USTAR_REGULAR = '0', /* Ordinary file. */ + USTAR_DIRECTORY = '5', /* Directory. */ + USTAR_EOF = -1 /* End of archive (not an official value). */ + }; + +/* Size of a ustar archive header, in bytes. */ +#define USTAR_HEADER_SIZE 512 + +bool ustar_make_header (const char *file_name, enum ustar_type, + int size, char header[USTAR_HEADER_SIZE]); +const char *ustar_parse_header (const char header[USTAR_HEADER_SIZE], + const char **file_name, + enum ustar_type *, int *size); + +#endif /* lib/ustar.h */ diff --git a/src/tests/filesys/extended/tar.c b/src/tests/filesys/extended/tar.c index 6f70d97..9801484 100644 --- a/src/tests/filesys/extended/tar.c +++ b/src/tests/filesys/extended/tar.c @@ -2,6 +2,7 @@ Creates a tar archive. */ +#include #include #include #include @@ -39,8 +40,7 @@ static bool archive_ordinary_file (const char *file_name, int file_fd, int archive_fd, bool *write_error); static bool archive_directory (char file_name[], size_t file_name_size, int file_fd, int archive_fd, bool *write_error); -static bool write_header (const char *file_name, - char type_flag, int size, unsigned mode, +static bool write_header (const char *file_name, enum ustar_type, int size, int archive_fd, bool *write_error); static bool do_write (int fd, const char *buffer, int size, bool *write_error); @@ -128,7 +128,8 @@ archive_ordinary_file (const char *file_name, int file_fd, bool success = true; int file_size = filesize (file_fd); - if (!write_header (file_name, '0', file_size, 0644, archive_fd, write_error)) + if (!write_header (file_name, USTAR_REGULAR, file_size, + archive_fd, write_error)) return false; while (file_size > 0) @@ -169,7 +170,7 @@ archive_directory (char file_name[], size_t file_name_size, int file_fd, return false; } - if (!write_header (file_name, '5', 0, 0755, archive_fd, write_error)) + if (!write_header (file_name, USTAR_DIRECTORY, 0, archive_fd, write_error)) return false; file_name[dir_len] = '/'; @@ -182,55 +183,12 @@ archive_directory (char file_name[], size_t file_name_size, int file_fd, } static bool -write_header (const char *file_name, - char type_flag, int size, unsigned mode, +write_header (const char *file_name, enum ustar_type type, int size, int archive_fd, bool *write_error) { static char header[512]; - unsigned chksum; - size_t i; - - memset (header, 0, sizeof header); - - /* Drop confusing and possibly dangerous prefixes from - FILE_NAME. */ - while (*file_name == '/' - || !memcmp (file_name, "./", 2) - || !memcmp (file_name, "../", 3)) - file_name = strchr (file_name, '/') + 1; - if (*file_name == '\0') - { - /* Dropped *everything* from FILE_NAME. - Should only be possible for a directory. */ - ASSERT (type_flag == '5'); - return true; - } - else if (strlen (file_name) > 99) - { - printf ("%s: file name too long\n", file_name); - return false; - } - - /* Fill in header except for final checksum. */ - strlcpy (header, file_name, 100); /* name */ - snprintf (header + 100, 8, "%07o", mode); /* mode */ - strlcpy (header + 108, "0000000", 8); /* uid */ - strlcpy (header + 116, "0000000", 8); /* gid */ - snprintf (header + 124, 12, "%011o", size); /* size */ - snprintf (header + 136, 12, "%011o", 1136102400); /* mtime (2006-01-01) */ - memset (header + 148, ' ', 8); /* chksum */ - header[156] = type_flag; /* typeflag */ - strlcpy (header + 257, "ustar", 6); /* magic */ - strlcpy (header + 263, "00", 3); /* version */ - - /* Compute and fill in final checksum. */ - chksum = 0; - for (i = 0; i < 512; i++) - chksum += (uint8_t) header[i]; - snprintf (header + 148, 8, "%07o", chksum); - - /* Write header. */ - return do_write (archive_fd, header, 512, write_error); + return (ustar_make_header (file_name, type, size, header) + && do_write (archive_fd, header, 512, write_error)); } static bool diff --git a/src/tests/tests.pm b/src/tests/tests.pm index 29e0707..4599cb9 100644 --- a/src/tests/tests.pm +++ b/src/tests/tests.pm @@ -561,12 +561,15 @@ sub read_tar { $size = 0 if $typeflag eq '5'; # Store content. + $name =~ s%^(/|\./|\.\./)*%%; # Strip leading "/", "./", "../". + $name = '' if $name eq '.' || $name eq '..'; if (exists $content{$name}) { fail "$archive: contains multiple entries for $name\n"; } if ($typeflag eq '5') { - $content{$name} = 'directory'; + $content{$name} = 'directory' if $name ne ''; } else { + fail "$archive: contains file with empty name\n" if $name eq ''; my ($position) = sysseek (ARCHIVE, 0, SEEK_CUR); $content{$name} = [$archive, $position, $size]; sysseek (ARCHIVE, int (($size + 511) / 512) * 512, SEEK_CUR); diff --git a/src/threads/init.c b/src/threads/init.c index b89b282..bb7e301 100644 --- a/src/threads/init.c +++ b/src/threads/init.c @@ -302,8 +302,8 @@ run_actions (char **argv) {"ls", 1, fsutil_ls}, {"cat", 2, fsutil_cat}, {"rm", 2, fsutil_rm}, - {"put", 2, fsutil_put}, - {"get", 2, fsutil_get}, + {"extract", 1, fsutil_extract}, + {"append", 2, fsutil_append}, #endif {NULL, 0, NULL}, }; @@ -351,8 +351,8 @@ usage (void) " cat FILE Print FILE to the console.\n" " rm FILE Delete FILE.\n" "Use these actions indirectly via `pintos' -g and -p options:\n" - " put FILE Put FILE into file system from scratch disk.\n" - " get FILE Get FILE from file system into scratch disk.\n" + " extract Untar from scratch disk into file system.\n" + " append FILE Append FILE to tar file on scratch disk.\n" #endif "\nOptions:\n" " -h Print this help message and power off.\n" diff --git a/src/utils/pintos b/src/utils/pintos index 88bd3ad..0b48be1 100755 --- a/src/utils/pintos +++ b/src/utils/pintos @@ -252,8 +252,15 @@ sub find_disks { # Prepare the scratch disk for gets and puts. sub prepare_scratch_disk { - # Copy the files to put onto the scratch disk. - put_scratch_file ($_->[0]) foreach @puts; + if (@puts) { + # Write ustar header and data for each file. + put_scratch_file ($_->[0], + defined $_->[1] ? $_->[1] : $_->[0]) + foreach @puts; + + # Write end-of-archive marker. + print { $disks{SCRATCH}{HANDLE} } "\0" x 1024; + } # Make sure the scratch disk is big enough to get big files. extend_disk ($disks{SCRATCH}, @gets * 1024 * 1024) if @gets; @@ -272,35 +279,86 @@ sub finish_scratch_disk { my ($ok) = 1; foreach my $get (@gets) { my ($name) = defined ($get->[1]) ? $get->[1] : $get->[0]; - $ok &&= get_scratch_file ($name); - if (!$ok) { + my ($error) = get_scratch_file ($name); + if ($error) { + print STDERR "getting $name failed ($error)\n"; die "$name: unlink: $!\n" if !unlink ($name) && !$!{ENOENT}; + $ok = 0; } } } -# put_scratch_file($file). +# mk_ustar_field($number, $size) +# +# Returns $number in a $size-byte numeric field in the format used by +# the standard ustar archive header. +sub mk_ustar_field { + my ($number, $size) = @_; + my ($len) = $size - 1; + my ($out) = sprintf ("%0${len}o", $number) . "\0"; + die "$number: too large for $size-byte octal ustar field\n" + if length ($out) != $size; + return $out; +} + +# calc_ustar_chksum($s) +# +# Calculates and returns the ustar checksum of 512-byte ustar archive +# header $s. +sub calc_ustar_chksum { + my ($s) = @_; + die if length ($s) != 512; + substr ($s, 148, 8, ' ' x 8); + return unpack ("%32a*", $s); +} + +# put_scratch_file($src_file_name, $dst_file_name). # -# Copies $file into the scratch disk. +# Copies $src_file_name into the scratch disk for extraction as +# $dst_file_name. sub put_scratch_file { - my ($put_file_name) = @_; + my ($src_file_name, $dst_file_name) = @_; my ($disk_handle, $disk_file_name) = open_disk ($disks{SCRATCH}); - print "Copying $put_file_name into $disk_file_name...\n"; + print "Copying $src_file_name to scratch partition...\n"; + + # ustar format supports up to 100 characters for a file name, and + # even longer names given some common properties, but our code in + # the Pintos kernel only supports at most 99 characters. + die "$dst_file_name: name too long (max 99 characters)\n" + if length ($dst_file_name) > 99; - # Write metadata sector, which consists of a 4-byte signature - # followed by the file size. - stat $put_file_name or die "$put_file_name: stat: $!\n"; + # Compose and write ustar header. + stat $src_file_name or die "$src_file_name: stat: $!\n"; my ($size) = -s _; - my ($metadata) = pack ("a4 V x504", "PUT\0", $size); - write_fully ($disk_handle, $disk_file_name, $metadata); + my ($header) = (pack ("a100", $dst_file_name) # name + . mk_ustar_field (0644, 8) # mode + . mk_ustar_field (0, 8) # uid + . mk_ustar_field (0, 8) # gid + . mk_ustar_field ($size, 12) # size + . mk_ustar_field (1136102400, 12) # mtime + . (' ' x 8) # chksum + . '0' # typeflag + . ("\0" x 100) # linkname + . "ustar\0" # magic + . "00" # version + . "root" . ("\0" x 28) # uname + . "root" . ("\0" x 28) # gname + . "\0" x 8 # devmajor + . "\0" x 8 # devminor + . ("\0" x 155)) # prefix + . "\0" x 12; # pad to 512 bytes + substr ($header, 148, 8) = mk_ustar_field (calc_ustar_chksum ($header), 8); + write_fully ($disk_handle, $disk_file_name, $header); # Copy file data. my ($put_handle); - sysopen ($put_handle, $put_file_name, O_RDONLY) - or die "$put_file_name: open: $!\n"; - copy_file ($put_handle, $put_file_name, $disk_handle, $disk_file_name, + sysopen ($put_handle, $src_file_name, O_RDONLY) + or die "$src_file_name: open: $!\n"; + copy_file ($put_handle, $src_file_name, $disk_handle, $disk_file_name, $size); + die "$src_file_name: changed size while being read\n" + if $size != -s $put_handle; close ($put_handle); # Round up disk data to beginning of next sector. @@ -318,13 +376,27 @@ sub get_scratch_file { print "Copying $get_file_name out of $disk_file_name...\n"; - # Read metadata sector, which has a 4-byte signature followed by - # the file size. - my ($metadata) = read_fully ($disk_handle, $disk_file_name, 512); - my ($signature, $size) = unpack ("a4 V", $metadata); - (print STDERR "bad signature on scratch disk--did Pintos run fail?\n"), - return 0 - if $signature ne "GET\0"; + # Read ustar header sector. + my ($header) = read_fully ($disk_handle, $disk_file_name, 512); + return "scratch disk tar archive ends unexpectedly" + if $header eq ("\0" x 512); + + # Verify magic numbers. + return "corrupt ustar signature" if substr ($header, 257, 6) ne "ustar\0"; + return "invalid ustar version" if substr ($header, 263, 2) ne '00'; + + # Verify checksum. + my ($chksum) = oct (unpack ("Z*", substr ($header, 148, 8))); + my ($correct_chksum) = calc_ustar_chksum ($header); + return "checksum mismatch" if $chksum != $correct_chksum; + + # Get type. + my ($typeflag) = substr ($header, 156, 1); + return "not a regular file" if $typeflag ne '0' && $typeflag ne "\0"; + + # Get size. + my ($size) = oct (unpack ("Z*", substr ($header, 124, 12))); + return "bad size $size\n" if $size < 0; # Copy file data. my ($get_handle); @@ -338,7 +410,7 @@ sub get_scratch_file { read_fully ($disk_handle, $disk_file_name, 512 - $size % 512) if $size % 512; - return 1; + return 0; } # Prepares the arguments to pass to the Pintos kernel, @@ -347,9 +419,9 @@ sub prepare_arguments { my (@args); push (@args, shift (@kernel_args)) while @kernel_args && $kernel_args[0] =~ /^-/; - push (@args, 'put', defined $_->[1] ? $_->[1] : $_->[0]) foreach @puts; + push (@args, 'extract') if @puts; push (@args, @kernel_args); - push (@args, 'get', $_->[0]) foreach @gets; + push (@args, 'append', $_->[0]) foreach @gets; write_cmd_line ($disks{OS}, @args); }